21:03:15.071 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:03:15.071 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrEFwPR/test-tmp
21:03:15.071 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
21:03:15.071 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrEFwPR/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
21:03:15.072 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:03:15.378 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:03:15.378 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:03:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.380321 3213 mini_oidc.cc:150] Starting JWKS server
21:03:15.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.382915 3213 webserver.cc:458] Webserver started at http://127.0.0.1:41543/ using document root <none> and password file <none>
21:03:15.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.383117 3213 mini_oidc.cc:186] Starting OIDC Discovery server
21:03:15.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.383639 3213 webserver.cc:458] Webserver started at http://127.0.0.1:34789/ using document root <none> and password file <none>
21:03:15.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T21:03:15Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:03:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T21:03:15Z Disabled control of system clock
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.412297 3213 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/wal
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/data
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/logs
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/data/info.pb
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.126:35753
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.126
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41543/jwks.json
21:03:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:03:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:03:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.712467 3224 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.712822 3224 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.712960 3224 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.720276 3224 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:03:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.720408 3224 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:03:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.720467 3224 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.720537 3224 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.720597 3224 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:03:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.720678 3224 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.720755 3224 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41543/jwks.json
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.729858 3224 master_runner.cc:386] Master server non-default flags:
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/data
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/wal
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.126:35753
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41543/jwks.json
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/data/info.pb
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.126
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/logs
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:03:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:03:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 20:43:22 UTC on 24a791456cd2
21:03:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4163
21:03:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.731640 3224 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:15.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.733278 3224 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:15.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.743391 3232 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:15.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.743537 3229 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:15.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.743805 3230 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:15.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.744554 3224 server_base.cc:1034] running on GCE node
21:03:15.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.745304 3224 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:15.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.746631 3224 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:15.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.747834 3224 hybrid_clock.cc:648] HybridClock initialized: now 1736802195747801 us; error 34 us; skew 500 ppm
21:03:15.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.748210 3224 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.750355 3224 webserver.cc:458] Webserver started at http://127.3.35.126:32773/ using document root <none> and password file <none>
21:03:15.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.751072 3224 fs_manager.cc:362] Metadata directory not provided
21:03:15.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.751180 3224 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:15.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.751484 3224 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:03:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.754009 3224 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/data/instance:
21:03:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "368a4202fe704bd8addedbf58359d23f"
21:03:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:15 on dist-test-slave-wczx"
21:03:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.754746 3224 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/wal/instance:
21:03:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "368a4202fe704bd8addedbf58359d23f"
21:03:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:15 on dist-test-slave-wczx"
21:03:15.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.759567 3224 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.004s
21:03:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.762861 3238 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.764112 3224 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.764261 3224 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/data,/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/wal
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "368a4202fe704bd8addedbf58359d23f"
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:15 on dist-test-slave-wczx"
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.764456 3224 fs_report.cc:389] FS layout report
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/wal
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/wal
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/data/data
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.775615 3224 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.776432 3224 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.776701 3224 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.796859 3224 rpc_server.cc:307] RPC server started. Bound to: 127.3.35.126:35753
21:03:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.796892 3299 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.35.126:35753 every 8 connection(s)
21:03:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.798187 3224 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/data/info.pb
21:03:15.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.801684 3300 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:15.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.807657 3213 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu as pid 3224
21:03:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.807082 3300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } has no permanent_uuid. Determining permanent_uuid...
21:03:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.807961 3213 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-0/wal/instance
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.810524 3213 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/wal
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/data
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/logs
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/data/info.pb
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.125:42683
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.125
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41543/jwks.json
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:03:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:03:15.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.824932 3300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } has no permanent_uuid. Determining permanent_uuid...
21:03:15.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.826104 3241 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.35.125:42683: connect: Connection refused (error 111)
21:03:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.827507 3300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.125:42683: Network error: Client connection negotiation failed: client connection to 127.3.35.125:42683: connect: Connection refused (error 111)
21:03:15.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.876880 3300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } attempt: 1
21:03:15.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.878719 3300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.125:42683: Network error: Client connection negotiation failed: client connection to 127.3.35.125:42683: connect: Connection refused (error 111)
21:03:15.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:15.947165 3300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } attempt: 2
21:03:15.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:15.949071 3300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.125:42683: Network error: Client connection negotiation failed: client connection to 127.3.35.125:42683: connect: Connection refused (error 111)
21:03:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.040347 3300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } attempt: 3
21:03:16.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.042219 3300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.125:42683: Network error: Client connection negotiation failed: client connection to 127.3.35.125:42683: connect: Connection refused (error 111)
21:03:16.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.119189 3304 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:16.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.119488 3304 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:16.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.119585 3304 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.126921 3304 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:03:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.127017 3304 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:03:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.127051 3304 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.127096 3304 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.127151 3304 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:03:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.127200 3304 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.127238 3304 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41543/jwks.json
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.136556 3304 master_runner.cc:386] Master server non-default flags:
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/data
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/wal
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.125:42683
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41543/jwks.json
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/data/info.pb
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.125
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/logs
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 20:43:22 UTC on 24a791456cd2
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4163
21:03:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:16.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.138000 3304 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:16.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.139583 3304 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:16.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.149480 3310 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:16.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.149785 3313 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.150543 3311 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.151341 3304 server_base.cc:1034] running on GCE node
21:03:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.152122 3304 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:16.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.153514 3304 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:16.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.154702 3304 hybrid_clock.cc:648] HybridClock initialized: now 1736802196154658 us; error 35 us; skew 500 ppm
21:03:16.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.155019 3304 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.157087 3304 webserver.cc:458] Webserver started at http://127.3.35.125:37063/ using document root <none> and password file <none>
21:03:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.157783 3304 fs_manager.cc:362] Metadata directory not provided
21:03:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.157871 3304 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:16.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.158125 3304 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:03:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.160605 3304 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/data/instance:
21:03:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb03beba58e94739ba9391a7dac8aac8"
21:03:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:16 on dist-test-slave-wczx"
21:03:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.161341 3304 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/wal/instance:
21:03:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb03beba58e94739ba9391a7dac8aac8"
21:03:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:16 on dist-test-slave-wczx"
21:03:16.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.166275 3304 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
21:03:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.169371 3319 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.170625 3304 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
21:03:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.170760 3304 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/data,/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/wal
21:03:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb03beba58e94739ba9391a7dac8aac8"
21:03:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:16 on dist-test-slave-wczx"
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.170949 3304 fs_report.cc:389] FS layout report
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/wal
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/wal
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/data/data
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.185477 3300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } attempt: 4
21:03:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.187299 3300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.125:42683: Network error: Client connection negotiation failed: client connection to 127.3.35.125:42683: connect: Connection refused (error 111)
21:03:16.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.189787 3304 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:16.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.190562 3304 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:16.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.190805 3304 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.210114 3304 rpc_server.cc:307] RPC server started. Bound to: 127.3.35.125:42683
21:03:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.210145 3380 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.35.125:42683 every 8 connection(s)
21:03:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.211377 3304 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/data/info.pb
21:03:16.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.214910 3381 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.218356 3213 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu as pid 3304
21:03:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.218562 3213 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-1/wal/instance
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.219919 3213 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/wal
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/data
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/logs
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/data/info.pb
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.124:46649
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.124
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41543/jwks.json
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:03:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:03:16.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.221509 3381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.235409 3381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.244566 3381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.245553 3322 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.247068 3381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.124:46649: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.296361 3381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } attempt: 1
21:03:16.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.298272 3381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.124:46649: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.366467 3381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } attempt: 2
21:03:16.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.368400 3381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.124:46649: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.459640 3381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } attempt: 3
21:03:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.461580 3381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.124:46649: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.486608 3300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } attempt: 5
21:03:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.493290 3300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.494920 3300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.124:46649: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.538204 3383 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.538496 3383 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.538619 3383 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.546200 3383 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:03:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.546118 3300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } attempt: 1
21:03:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.546324 3383 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:03:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.546379 3383 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.546445 3383 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.546505 3383 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:03:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.546582 3383 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.546633 3383 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41543/jwks.json
21:03:16.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.547789 3300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.124:46649: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.555773 3383 master_runner.cc:386] Master server non-default flags:
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/data
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/wal
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.124:46649
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41543/jwks.json
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/data/info.pb
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.124
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/logs
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 20:43:22 UTC on 24a791456cd2
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4163
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.557722 3383 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.559414 3383 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.572429 3391 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.572724 3392 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.573364 3394 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.574105 3383 server_base.cc:1034] running on GCE node
21:03:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.574862 3383 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.576252 3383 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.577447 3383 hybrid_clock.cc:648] HybridClock initialized: now 1736802196577410 us; error 26 us; skew 500 ppm
21:03:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.577796 3383 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.579988 3383 webserver.cc:458] Webserver started at http://127.3.35.124:46357/ using document root <none> and password file <none>
21:03:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.580684 3383 fs_manager.cc:362] Metadata directory not provided
21:03:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.580801 3383 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:16.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.581101 3383 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:03:16.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.583572 3383 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/data/instance:
21:03:16.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8e1b47b9a32447d8758d067e7a5dd60"
21:03:16.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:16 on dist-test-slave-wczx"
21:03:16.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.584374 3383 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/wal/instance:
21:03:16.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8e1b47b9a32447d8758d067e7a5dd60"
21:03:16.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:16 on dist-test-slave-wczx"
21:03:16.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.588977 3383 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
21:03:16.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.592164 3400 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.593246 3383 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.593392 3383 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/data,/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/wal
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8e1b47b9a32447d8758d067e7a5dd60"
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:16 on dist-test-slave-wczx"
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.593590 3383 fs_report.cc:389] FS layout report
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/wal
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/wal
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/data/data
21:03:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:16.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.604779 3381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } attempt: 4
21:03:16.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.606437 3381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.124:46649: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.616029 3300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } attempt: 2
21:03:16.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.617573 3300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.35.124:46649: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:16.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.626739 3383 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:16.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.627542 3383 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:16.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.627815 3383 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:16.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.647940 3383 rpc_server.cc:307] RPC server started. Bound to: 127.3.35.124:46649
21:03:16.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.647994 3461 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.35.124:46649 every 8 connection(s)
21:03:16.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.649268 3383 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/data/info.pb
21:03:16.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.652948 3462 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.658247 3213 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu as pid 3383
21:03:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.658427 3213 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/master-2/wal/instance
21:03:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.658718 3462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.674876 3462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.682353 3462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.692633 3462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60: Bootstrap starting.
21:03:16.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.695999 3462 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60: Neither blocks nor log segments found. Creating new log.
21:03:16.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.696916 3462 log.cc:826] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60: Log is configured to *not* fsync() on all Append() calls
21:03:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.699554 3462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60: No bootstrap required, opened a new log
21:03:16.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.705865 3462 raft_consensus.cc:357] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.706162 3462 raft_consensus.cc:383] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:16.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.706284 3462 raft_consensus.cc:738] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c8e1b47b9a32447d8758d067e7a5dd60, State: Initialized, Role: FOLLOWER
21:03:16.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.706878 3462 consensus_queue.cc:260] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.707635 3471 sys_catalog.cc:455] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:16.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.707918 3471 sys_catalog.cc:458] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: This master's current role is: FOLLOWER
21:03:16.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.708689 3462 sys_catalog.cc:564] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: configured and running, proceeding with master startup.
21:03:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.717895 3482 catalog_manager.cc:1559] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:03:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.718024 3482 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:03:16.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.715355 3383 master_runner.cc:186] Error getting master registration for 127.3.35.126:35753: OK, instance_id { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" instance_seqno: 1736802195791823 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:16.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.721606 3383 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:03:16.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.723791 3300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } attempt: 3
21:03:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.732971 3300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: Bootstrap starting.
21:03:16.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.735834 3300 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: Neither blocks nor log segments found. Creating new log.
21:03:16.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.736800 3300 log.cc:826] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: Log is configured to *not* fsync() on all Append() calls
21:03:16.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.740139 3300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: No bootstrap required, opened a new log
21:03:16.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.746008 3300 raft_consensus.cc:357] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.746258 3300 raft_consensus.cc:383] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:16.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.746363 3300 raft_consensus.cc:738] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 368a4202fe704bd8addedbf58359d23f, State: Initialized, Role: FOLLOWER
21:03:16.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.746917 3300 consensus_queue.cc:260] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.747758 3485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:16.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.748088 3485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: This master's current role is: FOLLOWER
21:03:16.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.748817 3300 sys_catalog.cc:564] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: configured and running, proceeding with master startup.
21:03:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.758356 3496 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:03:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.758615 3496 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:03:16.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.761067 3471 raft_consensus.cc:491] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:16.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.761308 3471 raft_consensus.cc:513] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.762979 3471 leader_election.cc:290] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), fb03beba58e94739ba9391a7dac8aac8 (127.3.35.125:42683)
21:03:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.763607 3265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "368a4202fe704bd8addedbf58359d23f" is_pre_election: true
21:03:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.764205 3265 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c8e1b47b9a32447d8758d067e7a5dd60 in term 0.
21:03:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.765165 3401 leader_election.cc:304] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60; no voters:
21:03:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.758114 3224 master_runner.cc:186] Error getting master registration for 127.3.35.125:42683: OK, instance_id { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" instance_seqno: 1736802196205804 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.768121 3471 raft_consensus.cc:2798] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.768277 3471 raft_consensus.cc:491] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.768368 3471 raft_consensus.cc:3054] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 0 FOLLOWER]: Advancing to term 1
21:03:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.771492 3471 raft_consensus.cc:513] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.772207 3471 leader_election.cc:290] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [CANDIDATE]: Term 1 election: Requested vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), fb03beba58e94739ba9391a7dac8aac8 (127.3.35.125:42683)
21:03:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.772629 3224 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:03:16.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.772735 3265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "368a4202fe704bd8addedbf58359d23f"
21:03:16.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.772924 3265 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 0 FOLLOWER]: Advancing to term 1
21:03:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.775966 3265 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c8e1b47b9a32447d8758d067e7a5dd60 in term 1.
21:03:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.776374 3401 leader_election.cc:304] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60; no voters:
21:03:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.776680 3471 raft_consensus.cc:2798] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 1 FOLLOWER]: Leader election won for term 1
21:03:16.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:16.779003 3244 tablet.cc:2367] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:16.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.796160 3471 raft_consensus.cc:695] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [term 1 LEADER]: Becoming Leader. State: Replica: c8e1b47b9a32447d8758d067e7a5dd60, State: Running, Role: LEADER
21:03:16.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.796942 3471 consensus_queue.cc:237] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.819607 3498 sys_catalog.cc:455] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c8e1b47b9a32447d8758d067e7a5dd60. Latest consensus state: current_term: 1 leader_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.820097 3498 sys_catalog.cc:458] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: This master's current role is: LEADER
21:03:16.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.830637 3500 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:03:16.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.833006 3500 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:03:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.846467 3265 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 1 FOLLOWER]: Refusing update from remote peer c8e1b47b9a32447d8758d067e7a5dd60: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.847589 3471 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [LEADER]: Connected to new peer: Peer: permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.867583 3485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: SysCatalogTable state changed. Reason: New leader c8e1b47b9a32447d8758d067e7a5dd60. Latest consensus state: current_term: 1 leader_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.867915 3485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: This master's current role is: FOLLOWER
21:03:16.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.869182 3501 mvcc.cc:204] Tried to move back new op lower bound from 7113941798267977728 to 7113941798086975488. Current Snapshot: MvccSnapshot[applied={T|T < 7113941798267977728}]
21:03:16.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.881035 3498 sys_catalog.cc:455] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:16.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.881294 3498 sys_catalog.cc:458] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: This master's current role is: LEADER
21:03:16.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.882807 3500 catalog_manager.cc:1348] Generated new cluster ID: 2b1133ab643f46d48deaebf094430c50
21:03:16.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.882761 3485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:16.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.882920 3500 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:03:16.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.882979 3485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: This master's current role is: FOLLOWER
21:03:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.903671 3500 catalog_manager.cc:1371] Generated new certificate authority record
21:03:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.905598 3500 catalog_manager.cc:1505] Loading token signing keys...
21:03:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.905633 3381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } attempt: 5
21:03:16.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.919292 3381 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8: Bootstrap starting.
21:03:16.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.926729 3381 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8: Neither blocks nor log segments found. Creating new log.
21:03:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.929183 3381 log.cc:826] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8: Log is configured to *not* fsync() on all Append() calls
21:03:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.941772 3500 catalog_manager.cc:5899] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60: Generated new TSK 0
21:03:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.942945 3500 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:03:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.944602 3381 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8: No bootstrap required, opened a new log
21:03:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.960577 3381 raft_consensus.cc:357] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.961217 3381 raft_consensus.cc:383] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.961419 3381 raft_consensus.cc:738] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fb03beba58e94739ba9391a7dac8aac8, State: Initialized, Role: FOLLOWER
21:03:16.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.962639 3381 consensus_queue.cc:260] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.966859 3213 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/wal
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/data
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/logs
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/data/info.pb
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.65:0
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.35.65
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.65
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:03:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.968883 3345 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fb03beba58e94739ba9391a7dac8aac8"
21:03:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.969256 3344 raft_consensus.cc:3054] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 0 FOLLOWER]: Advancing to term 1
21:03:16.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.972846 3344 raft_consensus.cc:1270] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 1 FOLLOWER]: Refusing update from remote peer c8e1b47b9a32447d8758d067e7a5dd60: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:16.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.973089 3346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fb03beba58e94739ba9391a7dac8aac8" is_pre_election: true
21:03:16.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.974768 3498 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:16.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:16.980386 3381 sys_catalog.cc:564] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: configured and running, proceeding with master startup.
21:03:17.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.029762 3507 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:17.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.030150 3507 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: This master's current role is: FOLLOWER
21:03:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.031342 3515 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.031351 3509 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c8e1b47b9a32447d8758d067e7a5dd60. Latest consensus state: current_term: 1 leader_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.031965 3515 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: This master's current role is: FOLLOWER
21:03:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.031970 3509 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: This master's current role is: FOLLOWER
21:03:17.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.067242 3526 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:03:17.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.086481 3498 sys_catalog.cc:455] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:17.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.086791 3498 sys_catalog.cc:458] T 00000000000000000000000000000000 P c8e1b47b9a32447d8758d067e7a5dd60 [sys.catalog]: This master's current role is: LEADER
21:03:17.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.402815 3508 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:17.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.403120 3508 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:17.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.403240 3508 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.410681 3508 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.410818 3508 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:03:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.410856 3508 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:03:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.410902 3508 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:03:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.411007 3508 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.35.65
21:03:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.419493 3508 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/data
21:03:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/wal
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.65:0
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/data/info.pb
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.65
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.35.65
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/logs
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 20:43:22 UTC on 24a791456cd2
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4163
21:03:17.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.421061 3508 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.422688 3508 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:17.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.433614 3533 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.434165 3535 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.434567 3532 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.434973 3508 server_base.cc:1034] running on GCE node
21:03:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.435511 3508 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.436372 3508 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:17.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.437556 3508 hybrid_clock.cc:648] HybridClock initialized: now 1736802197437522 us; error 58 us; skew 500 ppm
21:03:17.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.437825 3508 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.444463 3508 webserver.cc:458] Webserver started at http://127.3.35.65:46125/ using document root <none> and password file <none>
21:03:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.445161 3508 fs_manager.cc:362] Metadata directory not provided
21:03:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.445242 3508 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.445511 3508 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:03:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.448046 3508 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/data/instance:
21:03:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f5934ecef1445d8840f9064ebcb78c8"
21:03:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:17 on dist-test-slave-wczx"
21:03:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.448706 3508 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/wal/instance:
21:03:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f5934ecef1445d8840f9064ebcb78c8"
21:03:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:17 on dist-test-slave-wczx"
21:03:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.453428 3508 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
21:03:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.456295 3541 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.457464 3508 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.457581 3508 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/data,/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/wal
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f5934ecef1445d8840f9064ebcb78c8"
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:17 on dist-test-slave-wczx"
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.457737 3508 fs_report.cc:389] FS layout report
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/wal
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/wal
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/data/data
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.480157 3508 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.480885 3508 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:17.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.481141 3508 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.483927 3508 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.484037 3508 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.484151 3508 ts_tablet_manager.cc:610] Registered 0 tablets
21:03:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.484210 3508 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.550182 3508 rpc_server.cc:307] RPC server started. Bound to: 127.3.35.65:42735
21:03:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.550257 3665 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.35.65:42735 every 8 connection(s)
21:03:17.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.551589 3508 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/data/info.pb
21:03:17.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.553771 3213 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu as pid 3508
21:03:17.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.553992 3213 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-0/wal/instance
21:03:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.562525 3213 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/wal
21:03:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/data
21:03:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/logs
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/data/info.pb
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.66:0
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.35.66
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.66
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:03:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.570104 3667 heartbeater.cc:346] Connected to a master server at 127.3.35.126:35753
21:03:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.570456 3667 heartbeater.cc:463] Registering TS with master...
21:03:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.571655 3667 heartbeater.cc:510] Master 127.3.35.126:35753 requested a full tablet report, sending...
21:03:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.573581 3668 heartbeater.cc:346] Connected to a master server at 127.3.35.125:42683
21:03:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.573729 3668 heartbeater.cc:463] Registering TS with master...
21:03:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.574118 3668 heartbeater.cc:510] Master 127.3.35.125:42683 requested a full tablet report, sending...
21:03:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.574312 3255 ts_manager.cc:194] Registered new tserver with Master: 1f5934ecef1445d8840f9064ebcb78c8 (127.3.35.65:42735)
21:03:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.575067 3666 heartbeater.cc:346] Connected to a master server at 127.3.35.124:46649
21:03:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.575194 3666 heartbeater.cc:463] Registering TS with master...
21:03:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.575673 3666 heartbeater.cc:510] Master 127.3.35.124:46649 requested a full tablet report, sending...
21:03:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.575788 3336 ts_manager.cc:194] Registered new tserver with Master: 1f5934ecef1445d8840f9064ebcb78c8 (127.3.35.65:42735)
21:03:17.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.577035 3417 ts_manager.cc:194] Registered new tserver with Master: 1f5934ecef1445d8840f9064ebcb78c8 (127.3.35.65:42735)
21:03:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.578297 3417 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.35.65:42563
21:03:17.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.761533 3496 catalog_manager.cc:1260] Loaded cluster ID: 2b1133ab643f46d48deaebf094430c50
21:03:17.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.761649 3496 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: loading cluster ID for follower catalog manager: success
21:03:17.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.765329 3496 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: acquiring CA information for follower catalog manager: success
21:03:17.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.767798 3496 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:17.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.884292 3671 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:17.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.884559 3671 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:17.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.884656 3671 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.892061 3671 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.892202 3671 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:03:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.892241 3671 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:03:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.892287 3671 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:03:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.892385 3671 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.35.66
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.901386 3671 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/data
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/wal
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.66:0
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/data/info.pb
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.66
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.35.66
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/logs
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 20:43:22 UTC on 24a791456cd2
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4163
21:03:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.902925 3671 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:17.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.904512 3671 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.915438 3676 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.915534 3677 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:17.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:17.916237 3679 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:17.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.916697 3671 server_base.cc:1034] running on GCE node
21:03:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.917687 3671 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.918479 3671 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.919682 3671 hybrid_clock.cc:648] HybridClock initialized: now 1736802197919612 us; error 62 us; skew 500 ppm
21:03:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.920010 3671 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:17.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.922103 3671 webserver.cc:458] Webserver started at http://127.3.35.66:38129/ using document root <none> and password file <none>
21:03:17.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.922816 3671 fs_manager.cc:362] Metadata directory not provided
21:03:17.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.922901 3671 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:17.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.923167 3671 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:03:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.925652 3671 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/data/instance:
21:03:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "570a42c93cdf4ce3a720b0c1944a7ba2"
21:03:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:17 on dist-test-slave-wczx"
21:03:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.926317 3671 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/wal/instance:
21:03:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "570a42c93cdf4ce3a720b0c1944a7ba2"
21:03:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:17 on dist-test-slave-wczx"
21:03:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.931174 3671 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
21:03:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.934264 3685 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.935415 3671 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.935550 3671 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/data,/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/wal
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "570a42c93cdf4ce3a720b0c1944a7ba2"
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:17 on dist-test-slave-wczx"
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.935705 3671 fs_report.cc:389] FS layout report
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/wal
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/wal
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/data/data
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.949823 3671 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.950627 3671 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:17.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.950932 3671 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.954079 3671 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.954294 3671 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
21:03:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.954399 3671 ts_tablet_manager.cc:610] Registered 0 tablets
21:03:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:17.954437 3671 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.010855 3671 rpc_server.cc:307] RPC server started. Bound to: 127.3.35.66:37797
21:03:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.010902 3806 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.35.66:37797 every 8 connection(s)
21:03:18.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.012097 3671 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/data/info.pb
21:03:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.015939 3808 heartbeater.cc:346] Connected to a master server at 127.3.35.126:35753
21:03:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.016414 3213 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu as pid 3671
21:03:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.016580 3213 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-1/wal/instance
21:03:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.016371 3808 heartbeater.cc:463] Registering TS with master...
21:03:18.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.020018 3808 heartbeater.cc:510] Master 127.3.35.126:35753 requested a full tablet report, sending...
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.020538 3213 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/wal
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/data
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/logs
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/data/info.pb
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:18.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.67:0
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.35.67
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.67
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.021231 3807 heartbeater.cc:346] Connected to a master server at 127.3.35.124:46649
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.021366 3807 heartbeater.cc:463] Registering TS with master...
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.021900 3807 heartbeater.cc:510] Master 127.3.35.124:46649 requested a full tablet report, sending...
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.022099 3255 ts_manager.cc:194] Registered new tserver with Master: 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797)
21:03:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.022720 3417 ts_manager.cc:194] Registered new tserver with Master: 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797)
21:03:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.023765 3417 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.35.66:45051
21:03:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.025276 3809 heartbeater.cc:346] Connected to a master server at 127.3.35.125:42683
21:03:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.025410 3809 heartbeater.cc:463] Registering TS with master...
21:03:18.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.025920 3809 heartbeater.cc:510] Master 127.3.35.125:42683 requested a full tablet report, sending...
21:03:18.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.026798 3336 ts_manager.cc:194] Registered new tserver with Master: 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797)
21:03:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.071965 3526 catalog_manager.cc:1260] Loaded cluster ID: 2b1133ab643f46d48deaebf094430c50
21:03:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.072098 3526 catalog_manager.cc:1553] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8: loading cluster ID for follower catalog manager: success
21:03:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.075645 3526 catalog_manager.cc:1575] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8: acquiring CA information for follower catalog manager: success
21:03:18.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.077900 3526 catalog_manager.cc:1603] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.337415 3812 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.337699 3812 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.337857 3812 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.345440 3812 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.345594 3812 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:03:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.345649 3812 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:03:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.345714 3812 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:03:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.345839 3812 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.35.67
21:03:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.354598 3812 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.35.84:40523
21:03:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/data
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/wal
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.35.67:0
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/data/info.pb
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.35.67
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.35.67
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/logs
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 20:43:22 UTC on 24a791456cd2
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4163
21:03:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.356298 3812 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.357837 3812 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:18.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.370388 3818 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:18.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.370859 3819 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:18.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.371966 3812 server_base.cc:1034] running on GCE node
21:03:18.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.372203 3821 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:18.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.372774 3812 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:18.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.373657 3812 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.374876 3812 hybrid_clock.cc:648] HybridClock initialized: now 1736802198374825 us; error 41 us; skew 500 ppm
21:03:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.375136 3812 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.377491 3812 webserver.cc:458] Webserver started at http://127.3.35.67:36923/ using document root <none> and password file <none>
21:03:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.378162 3812 fs_manager.cc:362] Metadata directory not provided
21:03:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.378247 3812 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.378505 3812 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:03:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.381018 3812 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/data/instance:
21:03:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a2853666d6744f0a21f8e5178e36a92"
21:03:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:18 on dist-test-slave-wczx"
21:03:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.381662 3812 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/wal/instance:
21:03:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a2853666d6744f0a21f8e5178e36a92"
21:03:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:18 on dist-test-slave-wczx"
21:03:18.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.386677 3812 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.001s sys 0.005s
21:03:18.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.389549 3827 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:18.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.390758 3812 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.390877 3812 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/data,/tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/wal
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a2853666d6744f0a21f8e5178e36a92"
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 21:03:18 on dist-test-slave-wczx"
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.391036 3812 fs_report.cc:389] FS layout report
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/wal
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/wal
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/data/data
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:18.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.415822 3812 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:18.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.416641 3812 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:18.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.416880 3812 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.420091 3812 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.420327 3812 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.420557 3812 ts_tablet_manager.cc:610] Registered 0 tablets
21:03:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.420631 3812 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:18.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.474048 3812 rpc_server.cc:307] RPC server started. Bound to: 127.3.35.67:43115
21:03:18.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.474093 3948 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.35.67:43115 every 8 connection(s)
21:03:18.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.475302 3812 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/data/info.pb
21:03:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.478818 3950 heartbeater.cc:346] Connected to a master server at 127.3.35.126:35753
21:03:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.479071 3950 heartbeater.cc:463] Registering TS with master...
21:03:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.480108 3950 heartbeater.cc:510] Master 127.3.35.126:35753 requested a full tablet report, sending...
21:03:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.481513 3255 ts_manager.cc:194] Registered new tserver with Master: 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:18.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.482007 3213 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu as pid 3812
21:03:18.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.482203 3213 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskrEFwPR/test-tmp/mini-kudu-cluster913085582085319500/ts-2/wal/instance
21:03:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.484539 3949 heartbeater.cc:346] Connected to a master server at 127.3.35.124:46649
21:03:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.484660 3949 heartbeater.cc:463] Registering TS with master...
21:03:18.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.485196 3949 heartbeater.cc:510] Master 127.3.35.124:46649 requested a full tablet report, sending...
21:03:18.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.485659 3951 heartbeater.cc:346] Connected to a master server at 127.3.35.125:42683
21:03:18.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.485785 3951 heartbeater.cc:463] Registering TS with master...
21:03:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.486104 3417 ts_manager.cc:194] Registered new tserver with Master: 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.486249 3951 heartbeater.cc:510] Master 127.3.35.125:42683 requested a full tablet report, sending...
21:03:18.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.487141 3336 ts_manager.cc:194] Registered new tserver with Master: 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:18.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.487139 3417 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.35.67:52091
21:03:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.490643 3213 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
21:03:18.492 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:03:18.492 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:03:18.493 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.35.126"
port: 35753
}
bound_http_address {
host: "127.3.35.126"
port: 32773
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.35.125"
port: 42683
}
bound_http_address {
host: "127.3.35.125"
port: 37063
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.35.124"
port: 46649
}
bound_http_address {
host: "127.3.35.124"
port: 46357
}
}
}
21:03:18.493 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:03:18.494 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.35.65"
port: 42735
}
bound_http_address {
host: "127.3.35.65"
port: 46125
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.35.66"
port: 37797
}
bound_http_address {
host: "127.3.35.66"
port: 38129
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.35.67"
port: 43115
}
bound_http_address {
host: "127.3.35.67"
port: 36923
}
}
}
21:03:18.494 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:03:18.495 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:03:18.497 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:18.498 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.35.125:42683(127.3.35.125:42683)] connecting to peer
21:03:18.499 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.35.124:46649(127.3.35.124:46649)] Successfully connected to peer
21:03:18.501 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.35.124
21:03:18.506 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:18.507 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.3.35.125:42683(127.3.35.125:42683)] Successfully connected to peer
21:03:18.508 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.35.125
21:03:18.514 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5c24b0f1, L:/127.0.0.1:49526 - R:/127.3.35.124:46649] using SASL/PLAIN
21:03:18.514 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.35.126:35753(127.3.35.126:35753)] Successfully connected to peer
21:03:18.516 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.35.126
21:03:18.520 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:03:18.521 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=24), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [9ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: OK, deferred=null) after lookup
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.522619 3417 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49526:
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:18.523 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x0cd3d473, L:/127.0.0.1:34482 - R:/127.3.35.125:42683] using SASL/PLAIN
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.525025 3417 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:03:18.527 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x29727aad, L:/127.0.0.1:55596 - R:/127.3.35.126:35753] using SASL/PLAIN
21:03:18.544 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 75b9477e724848ec843d4f4bdd19cdf3
21:03:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.580477 3666 heartbeater.cc:502] Master 127.3.35.124:46649 was elected leader, sending a full tablet report...
21:03:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.585575 3595 tablet_service.cc:1467] Processing CreateTablet for tablet 629b751628af443ca1cf7b7c7e1a7e57 (DEFAULT_TABLE table=txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:03:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.586279 3737 tablet_service.cc:1467] Processing CreateTablet for tablet 629b751628af443ca1cf7b7c7e1a7e57 (DEFAULT_TABLE table=txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:03:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.586225 3739 tablet_service.cc:1467] Processing CreateTablet for tablet 56f4ea9ce0054d5898b79cdba5d3c814 (DEFAULT_TABLE table=txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:03:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.587477 3595 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 629b751628af443ca1cf7b7c7e1a7e57. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.587080 3881 tablet_service.cc:1467] Processing CreateTablet for tablet 56f4ea9ce0054d5898b79cdba5d3c814 (DEFAULT_TABLE table=txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:03:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.587720 3739 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 56f4ea9ce0054d5898b79cdba5d3c814. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.588161 3737 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 629b751628af443ca1cf7b7c7e1a7e57. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.588339 3881 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 56f4ea9ce0054d5898b79cdba5d3c814. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.586509 3596 tablet_service.cc:1467] Processing CreateTablet for tablet 56f4ea9ce0054d5898b79cdba5d3c814 (DEFAULT_TABLE table=txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:03:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.589864 3596 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 56f4ea9ce0054d5898b79cdba5d3c814. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.590266 3880 tablet_service.cc:1467] Processing CreateTablet for tablet 629b751628af443ca1cf7b7c7e1a7e57 (DEFAULT_TABLE table=txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:03:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.590677 3880 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 629b751628af443ca1cf7b7c7e1a7e57. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:18.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.606400 3975 tablet_bootstrap.cc:492] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8: Bootstrap starting.
21:03:18.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.607842 3976 tablet_bootstrap.cc:492] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2: Bootstrap starting.
21:03:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.610146 3977 tablet_bootstrap.cc:492] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92: Bootstrap starting.
21:03:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.610911 3976 tablet_bootstrap.cc:654] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2: Neither blocks nor log segments found. Creating new log.
21:03:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.611438 3975 tablet_bootstrap.cc:654] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8: Neither blocks nor log segments found. Creating new log.
21:03:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.612031 3976 log.cc:826] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2: Log is configured to *not* fsync() on all Append() calls
21:03:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.612690 3975 log.cc:826] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8: Log is configured to *not* fsync() on all Append() calls
21:03:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.613579 3977 tablet_bootstrap.cc:654] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92: Neither blocks nor log segments found. Creating new log.
21:03:18.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.614581 3976 tablet_bootstrap.cc:492] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2: No bootstrap required, opened a new log
21:03:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.614722 3977 log.cc:826] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92: Log is configured to *not* fsync() on all Append() calls
21:03:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.614851 3976 ts_tablet_manager.cc:1397] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.005s
21:03:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.615413 3975 tablet_bootstrap.cc:492] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8: No bootstrap required, opened a new log
21:03:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.615726 3975 ts_tablet_manager.cc:1397] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.005s
21:03:18.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.621295 3977 tablet_bootstrap.cc:492] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92: No bootstrap required, opened a new log
21:03:18.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.621572 3977 ts_tablet_manager.cc:1397] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.009s
21:03:18.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.621800 3976 raft_consensus.cc:357] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.622118 3976 raft_consensus.cc:383] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:18.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.622229 3976 raft_consensus.cc:738] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 570a42c93cdf4ce3a720b0c1944a7ba2, State: Initialized, Role: FOLLOWER
21:03:18.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.622855 3976 consensus_queue.cc:260] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.623824 3807 heartbeater.cc:502] Master 127.3.35.124:46649 was elected leader, sending a full tablet report...
21:03:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.624469 3976 ts_tablet_manager.cc:1428] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2: Time spent starting tablet: real 0.010s user 0.008s sys 0.002s
21:03:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.629971 3977 raft_consensus.cc:357] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.630369 3977 raft_consensus.cc:383] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.630465 3977 raft_consensus.cc:738] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0a2853666d6744f0a21f8e5178e36a92, State: Initialized, Role: FOLLOWER
21:03:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.631187 3977 consensus_queue.cc:260] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.631487 3976 tablet_bootstrap.cc:492] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2: Bootstrap starting.
21:03:18.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.633605 3976 tablet_bootstrap.cc:654] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2: Neither blocks nor log segments found. Creating new log.
21:03:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.638887 3949 heartbeater.cc:502] Master 127.3.35.124:46649 was elected leader, sending a full tablet report...
21:03:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.639314 3976 tablet_bootstrap.cc:492] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2: No bootstrap required, opened a new log
21:03:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.639560 3976 ts_tablet_manager.cc:1397] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2: Time spent bootstrapping tablet: real 0.008s user 0.004s sys 0.002s
21:03:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.640233 3977 ts_tablet_manager.cc:1428] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92: Time spent starting tablet: real 0.019s user 0.009s sys 0.010s
21:03:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.640643 3977 tablet_bootstrap.cc:492] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92: Bootstrap starting.
21:03:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.640627 3976 raft_consensus.cc:357] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.641413 3976 raft_consensus.cc:383] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.641635 3976 raft_consensus.cc:738] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 570a42c93cdf4ce3a720b0c1944a7ba2, State: Initialized, Role: FOLLOWER
21:03:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.641996 3976 consensus_queue.cc:260] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.642408 3976 ts_tablet_manager.cc:1428] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
21:03:18.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.642520 3977 tablet_bootstrap.cc:654] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92: Neither blocks nor log segments found. Creating new log.
21:03:18.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.651443 3975 raft_consensus.cc:357] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.651986 3975 raft_consensus.cc:383] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.652096 3975 raft_consensus.cc:738] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1f5934ecef1445d8840f9064ebcb78c8, State: Initialized, Role: FOLLOWER
21:03:18.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.652907 3975 consensus_queue.cc:260] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.654659 3975 ts_tablet_manager.cc:1428] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8: Time spent starting tablet: real 0.039s user 0.004s sys 0.011s
21:03:18.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.659186 3975 tablet_bootstrap.cc:492] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8: Bootstrap starting.
21:03:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.661612 3975 tablet_bootstrap.cc:654] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8: Neither blocks nor log segments found. Creating new log.
21:03:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.663851 3977 tablet_bootstrap.cc:492] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92: No bootstrap required, opened a new log
21:03:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.664054 3977 ts_tablet_manager.cc:1397] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92: Time spent bootstrapping tablet: real 0.024s user 0.006s sys 0.013s
21:03:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.664173 3975 tablet_bootstrap.cc:492] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8: No bootstrap required, opened a new log
21:03:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.664312 3975 ts_tablet_manager.cc:1397] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
21:03:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.664893 3977 raft_consensus.cc:357] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.665143 3977 raft_consensus.cc:383] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.665091 3975 raft_consensus.cc:357] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.665234 3977 raft_consensus.cc:738] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0a2853666d6744f0a21f8e5178e36a92, State: Initialized, Role: FOLLOWER
21:03:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.665298 3975 raft_consensus.cc:383] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.665378 3975 raft_consensus.cc:738] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1f5934ecef1445d8840f9064ebcb78c8, State: Initialized, Role: FOLLOWER
21:03:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.665499 3977 consensus_queue.cc:260] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.665578 3975 consensus_queue.cc:260] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.666091 3975 ts_tablet_manager.cc:1428] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
21:03:18.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.668877 3977 ts_tablet_manager.cc:1428] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
21:03:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.727627 3952 tablet.cc:2367] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.768841 3810 tablet.cc:2367] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.784835 3985 raft_consensus.cc:491] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.785063 3985 raft_consensus.cc:513] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.786530 3985 leader_election.cc:290] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797), 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.793854 3759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "629b751628af443ca1cf7b7c7e1a7e57" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" is_pre_election: true
21:03:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.794426 3759 raft_consensus.cc:2463] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 0.
21:03:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.794302 3904 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "629b751628af443ca1cf7b7c7e1a7e57" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0a2853666d6744f0a21f8e5178e36a92" is_pre_election: true
21:03:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.794785 3904 raft_consensus.cc:2463] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 0.
21:03:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.795037 3542 leader_election.cc:304] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f5934ecef1445d8840f9064ebcb78c8, 570a42c93cdf4ce3a720b0c1944a7ba2; no voters:
21:03:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.795420 3985 raft_consensus.cc:2798] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.795564 3985 raft_consensus.cc:491] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.795672 3985 raft_consensus.cc:3054] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Advancing to term 1
21:03:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.797626 3985 raft_consensus.cc:513] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.798223 3985 leader_election.cc:290] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 election: Requested vote from peers 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797), 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.798699 3904 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "629b751628af443ca1cf7b7c7e1a7e57" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0a2853666d6744f0a21f8e5178e36a92"
21:03:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.798810 3759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "629b751628af443ca1cf7b7c7e1a7e57" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2"
21:03:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.798888 3904 raft_consensus.cc:3054] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Advancing to term 1
21:03:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.799043 3759 raft_consensus.cc:3054] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Advancing to term 1
21:03:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.801273 3759 raft_consensus.cc:2463] T 629b751628af443ca1cf7b7c7e1a7e57 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 1.
21:03:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.801621 3542 leader_election.cc:304] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f5934ecef1445d8840f9064ebcb78c8, 570a42c93cdf4ce3a720b0c1944a7ba2; no voters:
21:03:18.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.801923 3985 raft_consensus.cc:2798] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 FOLLOWER]: Leader election won for term 1
21:03:18.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.802055 3904 raft_consensus.cc:2463] T 629b751628af443ca1cf7b7c7e1a7e57 P 0a2853666d6744f0a21f8e5178e36a92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 1.
21:03:18.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.804107 3985 raft_consensus.cc:695] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 LEADER]: Becoming Leader. State: Replica: 1f5934ecef1445d8840f9064ebcb78c8, State: Running, Role: LEADER
21:03:18.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.804606 3985 consensus_queue.cc:237] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.809718 3416 catalog_manager.cc:5526] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 reported cstate change: term changed from 0 to 1, leader changed from <none> to 1f5934ecef1445d8840f9064ebcb78c8 (127.3.35.65). New cstate: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } health_report { overall_health: UNKNOWN } } }
21:03:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:18.820966 3669 tablet.cc:2367] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:18.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.977035 3985 raft_consensus.cc:491] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:18.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.977280 3985 raft_consensus.cc:513] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.977834 3985 leader_election.cc:290] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797), 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.978305 3759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "56f4ea9ce0054d5898b79cdba5d3c814" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" is_pre_election: true
21:03:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.978412 3904 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "56f4ea9ce0054d5898b79cdba5d3c814" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0a2853666d6744f0a21f8e5178e36a92" is_pre_election: true
21:03:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.978596 3759 raft_consensus.cc:2463] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 0.
21:03:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.978653 3904 raft_consensus.cc:2463] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 0.
21:03:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.979033 3542 leader_election.cc:304] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0a2853666d6744f0a21f8e5178e36a92, 1f5934ecef1445d8840f9064ebcb78c8; no voters:
21:03:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.979295 3985 raft_consensus.cc:2798] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.979387 3985 raft_consensus.cc:491] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.979516 3985 raft_consensus.cc:3054] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Advancing to term 1
21:03:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.981441 3985 raft_consensus.cc:513] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.982010 3985 leader_election.cc:290] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 election: Requested vote from peers 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797), 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.982426 3759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "56f4ea9ce0054d5898b79cdba5d3c814" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2"
21:03:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.982591 3759 raft_consensus.cc:3054] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Advancing to term 1
21:03:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.982517 3904 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "56f4ea9ce0054d5898b79cdba5d3c814" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0a2853666d6744f0a21f8e5178e36a92"
21:03:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.982738 3904 raft_consensus.cc:3054] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Advancing to term 1
21:03:18.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.984481 3759 raft_consensus.cc:2463] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 1.
21:03:18.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.984608 3904 raft_consensus.cc:2463] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 1.
21:03:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.984905 3542 leader_election.cc:304] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f5934ecef1445d8840f9064ebcb78c8, 570a42c93cdf4ce3a720b0c1944a7ba2; no voters:
21:03:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.985175 3985 raft_consensus.cc:2798] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 FOLLOWER]: Leader election won for term 1
21:03:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.985683 3985 raft_consensus.cc:695] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 LEADER]: Becoming Leader. State: Replica: 1f5934ecef1445d8840f9064ebcb78c8, State: Running, Role: LEADER
21:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.985888 3985 consensus_queue.cc:237] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:18.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:18.988605 3416 catalog_manager.cc:5526] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 reported cstate change: term changed from 0 to 1, leader changed from <none> to 1f5934ecef1445d8840f9064ebcb78c8 (127.3.35.65). New cstate: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } health_report { overall_health: UNKNOWN } } }
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.077989 3416 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49534:
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:03:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:03:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:19.079121 3416 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:03:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.090188 3737 tablet_service.cc:1467] Processing CreateTablet for tablet fedb2ad0b5ca4d12b21c510aae95f278 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7cc272028a224994863b6b73755b3f35]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:03:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.090183 3596 tablet_service.cc:1467] Processing CreateTablet for tablet fedb2ad0b5ca4d12b21c510aae95f278 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7cc272028a224994863b6b73755b3f35]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:03:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.090268 3880 tablet_service.cc:1467] Processing CreateTablet for tablet fedb2ad0b5ca4d12b21c510aae95f278 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7cc272028a224994863b6b73755b3f35]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:03:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.091161 3880 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fedb2ad0b5ca4d12b21c510aae95f278. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.091189 3596 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fedb2ad0b5ca4d12b21c510aae95f278. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.091194 3737 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fedb2ad0b5ca4d12b21c510aae95f278. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.096226 3977 tablet_bootstrap.cc:492] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: Bootstrap starting.
21:03:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.097618 3977 tablet_bootstrap.cc:654] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: Neither blocks nor log segments found. Creating new log.
21:03:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.097903 3976 tablet_bootstrap.cc:492] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: Bootstrap starting.
21:03:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.097995 3975 tablet_bootstrap.cc:492] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: Bootstrap starting.
21:03:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.099364 3976 tablet_bootstrap.cc:654] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: Neither blocks nor log segments found. Creating new log.
21:03:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.099609 3975 tablet_bootstrap.cc:654] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: Neither blocks nor log segments found. Creating new log.
21:03:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.100085 3977 tablet_bootstrap.cc:492] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: No bootstrap required, opened a new log
21:03:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.100241 3977 ts_tablet_manager.cc:1397] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.002s
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.100980 3977 raft_consensus.cc:357] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.101235 3977 raft_consensus.cc:383] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.101316 3977 raft_consensus.cc:738] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0a2853666d6744f0a21f8e5178e36a92, State: Initialized, Role: FOLLOWER
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.101591 3976 tablet_bootstrap.cc:492] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: No bootstrap required, opened a new log
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.101567 3977 consensus_queue.cc:260] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.101706 3975 tablet_bootstrap.cc:492] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: No bootstrap required, opened a new log
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.101890 3975 ts_tablet_manager.cc:1397] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.102012 3977 ts_tablet_manager.cc:1428] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.101990 3982 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.102270 3982 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.102695 3975 raft_consensus.cc:357] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.102962 3975 raft_consensus.cc:383] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.103152 3975 raft_consensus.cc:738] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1f5934ecef1445d8840f9064ebcb78c8, State: Initialized, Role: FOLLOWER
21:03:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.103423 3975 consensus_queue.cc:260] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.103900 3985 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.104156 3985 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.101768 3976 ts_tablet_manager.cc:1397] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.105335 3975 ts_tablet_manager.cc:1428] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.108562 3976 raft_consensus.cc:357] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.108765 3976 raft_consensus.cc:383] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.108847 3976 raft_consensus.cc:738] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 570a42c93cdf4ce3a720b0c1944a7ba2, State: Initialized, Role: FOLLOWER
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.109102 3976 consensus_queue.cc:260] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.109522 3981 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.109889 3981 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:19.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.111855 3976 ts_tablet_manager.cc:1428] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
21:03:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.160419 3985 raft_consensus.cc:491] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.160610 3985 raft_consensus.cc:513] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.161360 3985 leader_election.cc:290] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797), 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.161866 3759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fedb2ad0b5ca4d12b21c510aae95f278" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" is_pre_election: true
21:03:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.161942 3904 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fedb2ad0b5ca4d12b21c510aae95f278" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0a2853666d6744f0a21f8e5178e36a92" is_pre_election: true
21:03:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.162106 3759 raft_consensus.cc:2463] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 0.
21:03:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.162163 3904 raft_consensus.cc:2463] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 0.
21:03:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.162540 3542 leader_election.cc:304] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f5934ecef1445d8840f9064ebcb78c8, 570a42c93cdf4ce3a720b0c1944a7ba2; no voters:
21:03:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.162802 3985 raft_consensus.cc:2798] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.162923 3985 raft_consensus.cc:491] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.162990 3985 raft_consensus.cc:3054] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 0 FOLLOWER]: Advancing to term 1
21:03:19.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.164942 3985 raft_consensus.cc:513] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.165371 3985 leader_election.cc:290] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 election: Requested vote from peers 570a42c93cdf4ce3a720b0c1944a7ba2 (127.3.35.66:37797), 0a2853666d6744f0a21f8e5178e36a92 (127.3.35.67:43115)
21:03:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.165843 3759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fedb2ad0b5ca4d12b21c510aae95f278" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2"
21:03:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.165912 3904 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fedb2ad0b5ca4d12b21c510aae95f278" candidate_uuid: "1f5934ecef1445d8840f9064ebcb78c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0a2853666d6744f0a21f8e5178e36a92"
21:03:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.166057 3759 raft_consensus.cc:3054] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 0 FOLLOWER]: Advancing to term 1
21:03:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.166103 3904 raft_consensus.cc:3054] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92 [term 0 FOLLOWER]: Advancing to term 1
21:03:19.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.168815 3904 raft_consensus.cc:2463] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 1.
21:03:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.168803 3759 raft_consensus.cc:2463] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f5934ecef1445d8840f9064ebcb78c8 in term 1.
21:03:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.169193 3542 leader_election.cc:304] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f5934ecef1445d8840f9064ebcb78c8, 570a42c93cdf4ce3a720b0c1944a7ba2; no voters:
21:03:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.169449 3985 raft_consensus.cc:2798] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 FOLLOWER]: Leader election won for term 1
21:03:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.169582 3985 raft_consensus.cc:695] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [term 1 LEADER]: Becoming Leader. State: Replica: 1f5934ecef1445d8840f9064ebcb78c8, State: Running, Role: LEADER
21:03:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.169859 3985 consensus_queue.cc:237] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } }
21:03:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.172171 3989 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: TxnStatusTablet state changed. Reason: New leader 1f5934ecef1445d8840f9064ebcb78c8. Latest consensus state: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.172358 3989 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: This TxnStatusTablet replica's current role is: LEADER
21:03:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.172600 3416 catalog_manager.cc:5526] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 reported cstate change: term changed from 0 to 1, leader changed from <none> to 1f5934ecef1445d8840f9064ebcb78c8 (127.3.35.65). New cstate: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } health_report { overall_health: UNKNOWN } } }
21:03:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.172904 4007 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
21:03:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.173213 4007 txn_status_manager.cc:930] Loading transaction status metadata into memory...
21:03:19.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.205528 3904 raft_consensus.cc:1270] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92 [term 1 FOLLOWER]: Refusing update from remote peer 1f5934ecef1445d8840f9064ebcb78c8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:19.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.205528 3759 raft_consensus.cc:1270] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 1 FOLLOWER]: Refusing update from remote peer 1f5934ecef1445d8840f9064ebcb78c8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:19.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.206444 3985 consensus_queue.cc:1035] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:19.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.206753 3989 consensus_queue.cc:1035] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.228607 3981 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: TxnStatusTablet state changed. Reason: New leader 1f5934ecef1445d8840f9064ebcb78c8. Latest consensus state: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.228896 3981 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.233984 4011 mvcc.cc:204] Tried to move back new op lower bound from 7113941807936372736 to 7113941807802392576. Current Snapshot: MvccSnapshot[applied={T|T < 7113941807936372736}]
21:03:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.234957 3989 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.235508 3989 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: This TxnStatusTablet replica's current role is: LEADER
21:03:19.247 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
21:03:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.245085 3982 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: TxnStatusTablet state changed. Reason: New leader 1f5934ecef1445d8840f9064ebcb78c8. Latest consensus state: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.245325 3982 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.248211 3981 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.248425 3981 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 570a42c93cdf4ce3a720b0c1944a7ba2: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.249555 3985 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.249804 3985 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 1f5934ecef1445d8840f9064ebcb78c8: This TxnStatusTablet replica's current role is: LEADER
21:03:19.252 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 75b9477e724848ec843d4f4bdd19cdf3
21:03:19.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 56f4ea9ce0054d5898b79cdba5d3c814 for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
21:03:19.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 629b751628af443ca1cf7b7c7e1a7e57 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
21:03:19.257 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=56f4ea9ce0054d5898b79cdba5d3c814@[0a2853666d6744f0a21f8e5178e36a92(127.3.35.67:43115),1f5934ecef1445d8840f9064ebcb78c8(127.3.35.65:42735)[L],570a42c93cdf4ce3a720b0c1944a7ba2(127.3.35.66:37797)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=629b751628af443ca1cf7b7c7e1a7e57@[0a2853666d6744f0a21f8e5178e36a92(127.3.35.67:43115),1f5934ecef1445d8840f9064ebcb78c8(127.3.35.65:42735)[L],570a42c93cdf4ce3a720b0c1944a7ba2(127.3.35.66:37797)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
21:03:19.261 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.35.124:46649, [3ms] Sub RPC GetTableLocations: received response from server master-127.3.35.124:46649: OK, deferred=null) after lookup
21:03:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.258705 3982 tablet_replica.cc:440] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1f5934ecef1445d8840f9064ebcb78c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1f5934ecef1445d8840f9064ebcb78c8" member_type: VOTER last_known_addr { host: "127.3.35.65" port: 42735 } } peers { permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 } } peers { permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 } } }
21:03:19.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.258951 3982 tablet_replica.cc:443] T fedb2ad0b5ca4d12b21c510aae95f278 P 0a2853666d6744f0a21f8e5178e36a92: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:19.263 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1f5934ecef1445d8840f9064ebcb78c8(127.3.35.65:42735)] connecting to peer
21:03:19.264 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 1f5934ecef1445d8840f9064ebcb78c8(127.3.35.65:42735)] Successfully connected to peer
21:03:19.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.274277 3989 consensus_queue.cc:1035] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:19.278 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x93c6d5cd, L:/127.0.0.1:40742 - R:/127.3.35.65:42735] using TOKEN/null
21:03:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.290611 3989 consensus_queue.cc:1035] T 629b751628af443ca1cf7b7c7e1a7e57 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.317955 3759 raft_consensus.cc:1270] T 56f4ea9ce0054d5898b79cdba5d3c814 P 570a42c93cdf4ce3a720b0c1944a7ba2 [term 1 FOLLOWER]: Refusing update from remote peer 1f5934ecef1445d8840f9064ebcb78c8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.318017 3904 raft_consensus.cc:1270] T 56f4ea9ce0054d5898b79cdba5d3c814 P 0a2853666d6744f0a21f8e5178e36a92 [term 1 FOLLOWER]: Refusing update from remote peer 1f5934ecef1445d8840f9064ebcb78c8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.318570 4020 consensus_queue.cc:1035] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "570a42c93cdf4ce3a720b0c1944a7ba2" member_type: VOTER last_known_addr { host: "127.3.35.66" port: 37797 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.318858 3985 consensus_queue.cc:1035] T 56f4ea9ce0054d5898b79cdba5d3c814 P 1f5934ecef1445d8840f9064ebcb78c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0a2853666d6744f0a21f8e5178e36a92" member_type: VOTER last_known_addr { host: "127.3.35.67" port: 43115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:19.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.340617 4013 mvcc.cc:204] Tried to move back new op lower bound from 7113941808401559552 to 7113941807048060928. Current Snapshot: MvccSnapshot[applied={T|T < 7113941808401559552}]
21:03:19.340 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.35.124:46649
21:03:19.341 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:03:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.342024 3213 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu pid 3383
21:03:19.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.343307 3457 generic_service.cc:196] Checking for leaks (request via RPC)
21:03:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.520256 3213 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu with pid 3383
21:03:19.542 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.35.124:46649(127.3.35.124:46649)] handling channelInactive
21:03:19.542 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state READY due to: connection closed
21:03:19.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:19.542851 3239 connection.cc:537] server connection from 127.0.0.1:55576 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:03:19.761 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.762 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.762 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.762 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.3.35.124:46649(127.3.35.124:46649) for tablet Kudu Master: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.819 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.821192 4028 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c8e1b47b9a32447d8758d067e7a5dd60)
21:03:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.821419 4028 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:19.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.822304 4029 raft_consensus.cc:491] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c8e1b47b9a32447d8758d067e7a5dd60)
21:03:19.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.822481 4029 raft_consensus.cc:513] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:19.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.822729 4028 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:19.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.823314 3264 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fb03beba58e94739ba9391a7dac8aac8" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "368a4202fe704bd8addedbf58359d23f" is_pre_election: true
21:03:19.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.823580 3264 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fb03beba58e94739ba9391a7dac8aac8 in term 1.
21:03:19.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.823690 4029 leader_election.cc:290] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers fb03beba58e94739ba9391a7dac8aac8 (127.3.35.125:42683), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:19.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.824208 3320 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 368a4202fe704bd8addedbf58359d23f, fb03beba58e94739ba9391a7dac8aac8; no voters:
21:03:19.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.824391 3346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "368a4202fe704bd8addedbf58359d23f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fb03beba58e94739ba9391a7dac8aac8" is_pre_election: true
21:03:19.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.824618 3346 raft_consensus.cc:2463] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 368a4202fe704bd8addedbf58359d23f in term 1.
21:03:19.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.824795 4028 raft_consensus.cc:2798] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:03:19.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.825026 4028 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 1 FOLLOWER]: Starting leader election (detected failure of leader c8e1b47b9a32447d8758d067e7a5dd60)
21:03:19.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.825265 4028 raft_consensus.cc:3054] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 1 FOLLOWER]: Advancing to term 2
21:03:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.825367 3241 leader_election.cc:304] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 368a4202fe704bd8addedbf58359d23f, fb03beba58e94739ba9391a7dac8aac8; no voters:
21:03:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.825835 4029 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 1 FOLLOWER]: Leader pre-election won for term 2
21:03:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.825948 4029 raft_consensus.cc:491] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 1 FOLLOWER]: Starting leader election (detected failure of leader c8e1b47b9a32447d8758d067e7a5dd60)
21:03:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.826036 4029 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 1 FOLLOWER]: Advancing to term 2
21:03:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:19.826277 3241 leader_election.cc:336] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:19.826596 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:19.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.828400 4028 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.828835 4029 raft_consensus.cc:513] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.829399 3264 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fb03beba58e94739ba9391a7dac8aac8" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "368a4202fe704bd8addedbf58359d23f"
21:03:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.829504 4029 leader_election.cc:290] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 2 election: Requested vote from peers fb03beba58e94739ba9391a7dac8aac8 (127.3.35.125:42683), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.829684 3264 raft_consensus.cc:2388] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate fb03beba58e94739ba9391a7dac8aac8 in current term 2: Already voted for candidate 368a4202fe704bd8addedbf58359d23f in this term.
21:03:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.829695 4028 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 2 election: Requested vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.830152 3346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "368a4202fe704bd8addedbf58359d23f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fb03beba58e94739ba9391a7dac8aac8"
21:03:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.830432 3346 raft_consensus.cc:2388] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 368a4202fe704bd8addedbf58359d23f in current term 2: Already voted for candidate fb03beba58e94739ba9391a7dac8aac8 in this term.
21:03:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:19.830729 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.830835 3322 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.831122 4028 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:03:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:19.831374 3241 leader_election.cc:336] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.831483 3241 leader_election.cc:304] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 368a4202fe704bd8addedbf58359d23f; no voters: c8e1b47b9a32447d8758d067e7a5dd60, fb03beba58e94739ba9391a7dac8aac8
21:03:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:19.831707 4029 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:03:19.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.840 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.860 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:19.877293 3666 heartbeater.cc:643] Failed to heartbeat to 127.3.35.124:46649 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:19.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.939 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.940 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] sending RPC to server master-127.3.35.124:46649, [4ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [138ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@716194458(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@64ac2849 -> (continuation of Deferred@1502691234 after retry RPC after error@1072443116) -> (continuation of Deferred@400563993 after retry RPC after error@311133132) -> (continuation of Deferred@1110950416 after retry RPC after error@485625844) -> (continuation of Deferred@192795556 after retry RPC after error@1168549092) -> (continuation of Deferred@972933378 after retry RPC after error@897379430) -> (continuation of Deferred@1350171038 after retry RPC after error@545842929), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@77ea9e3b -> (continuation of Deferred@1502691234 after retry RPC after error@1072443116) -> (continuation of Deferred@400563993 after retry RPC after error@311133132) -> (continuation of Deferred@1110950416 after retry RPC after error@485625844) -> (continuation of Deferred@192795556 after retry RPC after error@1168549092) -> (continuation of Deferred@972933378 after retry RPC after error@897379430) -> (continuation of Deferred@1350171038 after retry RPC after error@545842929)))
21:03:19.941 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] sending RPC to server master-127.3.35.124:46649, [4ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [138ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@716194458(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$84/161789223@64ac2849 -> (continuation of Deferred@1502691234 after retry RPC after error@1072443116) -> (continuation of Deferred@400563993 after retry RPC after error@311133132) -> (continuation of Deferred@1110950416 after retry RPC after error@485625844) -> (continuation of Deferred@192795556 after retry RPC after error@1168549092) -> (continuation of Deferred@972933378 after retry RPC after error@897379430) -> (continuation of Deferred@1350171038 after retry RPC after error@545842929), errback=org.apache.kudu.client.KuduTransaction$$Lambda$85/213469261@77ea9e3b -> (continuation of Deferred@1502691234 after retry RPC after error@1072443116) -> (continuation of Deferred@400563993 after retry RPC after error@311133132) -> (continuation of Deferred@1110950416 after retry RPC after error@485625844) -> (continuation of Deferred@192795556 after retry RPC after error@1168549092) -> (continuation of Deferred@972933378 after retry RPC after error@897379430) -> (continuation of Deferred@1350171038 after retry RPC after error@545842929)))
21:03:19.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.960 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.960 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.960 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.980 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:19.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:19.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:19.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.000 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:20.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:20.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.020 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.020 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:20.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:20.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.039 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:20.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:20.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.059 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:20.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:20.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:20.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:20.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.138374 4029 raft_consensus.cc:491] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:20.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.138598 4029 raft_consensus.cc:513] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:20.140 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.139359 4029 leader_election.cc:290] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers fb03beba58e94739ba9391a7dac8aac8 (127.3.35.125:42683), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.140522 3346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "368a4202fe704bd8addedbf58359d23f" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fb03beba58e94739ba9391a7dac8aac8" is_pre_election: true
21:03:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.140805 3346 raft_consensus.cc:2463] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 368a4202fe704bd8addedbf58359d23f in term 2.
21:03:20.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.141069 3241 leader_election.cc:336] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:20.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.141353 3241 leader_election.cc:304] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 368a4202fe704bd8addedbf58359d23f, fb03beba58e94739ba9391a7dac8aac8; no voters: c8e1b47b9a32447d8758d067e7a5dd60
21:03:20.141 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:20.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.141722 4029 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 2 FOLLOWER]: Leader pre-election won for term 3
21:03:20.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.141862 4029 raft_consensus.cc:491] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:20.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.141983 4029 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 2 FOLLOWER]: Advancing to term 3
21:03:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.144107 4029 raft_consensus.cc:513] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.144655 4029 leader_election.cc:290] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 3 election: Requested vote from peers fb03beba58e94739ba9391a7dac8aac8 (127.3.35.125:42683), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:20.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.145125 3346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "368a4202fe704bd8addedbf58359d23f" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fb03beba58e94739ba9391a7dac8aac8"
21:03:20.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.145385 3346 raft_consensus.cc:3054] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 2 FOLLOWER]: Advancing to term 3
21:03:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.145967 3241 leader_election.cc:336] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.147485 3346 raft_consensus.cc:2463] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 368a4202fe704bd8addedbf58359d23f in term 3.
21:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.147922 3241 leader_election.cc:304] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 368a4202fe704bd8addedbf58359d23f, fb03beba58e94739ba9391a7dac8aac8; no voters: c8e1b47b9a32447d8758d067e7a5dd60
21:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.148216 4029 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 3 FOLLOWER]: Leader election won for term 3
21:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.148574 4029 raft_consensus.cc:695] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [term 3 LEADER]: Becoming Leader. State: Replica: 368a4202fe704bd8addedbf58359d23f, State: Running, Role: LEADER
21:03:20.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.148947 4029 consensus_queue.cc:237] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.150236 4037 sys_catalog.cc:455] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 368a4202fe704bd8addedbf58359d23f. Latest consensus state: current_term: 3 leader_uuid: "368a4202fe704bd8addedbf58359d23f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.150549 4037 sys_catalog.cc:458] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: This master's current role is: LEADER
21:03:20.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.151105 4038 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:03:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.153424 4038 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3]
21:03:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.153864 4038 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=7cc272028a224994863b6b73755b3f35]
21:03:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.155350 4038 tablet_loader.cc:96] loaded metadata for tablet 56f4ea9ce0054d5898b79cdba5d3c814 (table txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3])
21:03:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.155731 4038 tablet_loader.cc:96] loaded metadata for tablet 629b751628af443ca1cf7b7c7e1a7e57 (table txn_manager_fallback [id=75b9477e724848ec843d4f4bdd19cdf3])
21:03:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.156070 4038 tablet_loader.cc:96] loaded metadata for tablet fedb2ad0b5ca4d12b21c510aae95f278 (table kudu_system.kudu_transactions [id=7cc272028a224994863b6b73755b3f35])
21:03:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.156277 4038 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:03:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.157009 4038 catalog_manager.cc:1260] Loaded cluster ID: 2b1133ab643f46d48deaebf094430c50
21:03:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.157112 4038 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:03:20.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.158152 4038 catalog_manager.cc:1505] Loading token signing keys...
21:03:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.158933 4038 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f: Loaded TSK: 0
21:03:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.159726 4038 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:03:20.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:20.179 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.179 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.180 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:20.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:03:20.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=224), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, deferred=Deferred@2074789651(state=PENDING, result=null, callback=(continuation of Deferred@1122268648 after retry RPC after error@844365349) -> (continuation of Deferred@220357599 after retry RPC after error@547209015) -> (continuation of Deferred@1102566905 after retry RPC after error@1815656764) -> (continuation of Deferred@1922696191 after retry RPC after error@1068723868) -> (continuation of Deferred@1087663246 after retry RPC after error@1762446577) -> (continuation of Deferred@1261158356 after retry RPC after error@1867050915) -> (continuation of Deferred@203116248 after retry RPC after error@58683630) -> (continuation of Deferred@356297290 after retry RPC after error@1025600185), errback=(continuation of Deferred@1122268648 after retry RPC after error@844365349) -> (continuation of Deferred@220357599 after retry RPC after error@547209015) -> (continuation of Deferred@1102566905 after retry RPC after error@1815656764) -> (continuation of Deferred@1922696191 after retry RPC after error@1068723868) -> (continuation of Deferred@1087663246 after retry RPC after error@1762446577) -> (continuation of Deferred@1261158356 after retry RPC after error@1867050915) -> (continuation of Deferred@203116248 after retry RPC after error@58683630) -> (continuation of Deferred@356297290 after retry RPC after error@1025600185))) after lookup
21:03:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.208916 3285 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:55596 (request call id 17) took 25 ms (client timeout 25 ms). Trace:
21:03:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.209216 3285 rpcz_store.cc:269] 0113 21:03:20.183534 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:03:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 21:03:20.183621 (+ 87us) service_pool.cc:225] Handling call
21:03:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 21:03:20.208802 (+ 25181us) inbound_call.cc:173] Queueing success response
21:03:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":131,"init.queue_time_us":430,"thread_start_us":1229,"threads_started":5}
21:03:20.223 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=10, TimeoutTracker(timeout=250, elapsed=264), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: OK, [225ms] sending RPC to server master-127.3.35.126:35753, deferred=Deferred@2074789651(state=PENDING, result=null, callback=(continuation of Deferred@1122268648 after retry RPC after error@844365349) -> (continuation of Deferred@220357599 after retry RPC after error@547209015) -> (continuation of Deferred@1102566905 after retry RPC after error@1815656764) -> (continuation of Deferred@1922696191 after retry RPC after error@1068723868) -> (continuation of Deferred@1087663246 after retry RPC after error@1762446577) -> (continuation of Deferred@1261158356 after retry RPC after error@1867050915) -> (continuation of Deferred@203116248 after retry RPC after error@58683630) -> (continuation of Deferred@356297290 after retry RPC after error@1025600185) -> (continuation of Deferred@1375912265 after retry RPC@1562954967), errback=(continuation of Deferred@1122268648 after retry RPC after error@844365349) -> (continuation of Deferred@220357599 after retry RPC after error@547209015) -> (continuation of Deferred@1102566905 after retry RPC after error@1815656764) -> (continuation of Deferred@1922696191 after retry RPC after error@1068723868) -> (continuation of Deferred@1087663246 after retry RPC after error@1762446577) -> (continuation of Deferred@1261158356 after retry RPC after error@1867050915) -> (continuation of Deferred@203116248 after retry RPC after error@58683630) -> (continuation of Deferred@356297290 after retry RPC after error@1025600185) -> (continuation of Deferred@1375912265 after retry RPC@1562954967)))
21:03:20.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 906546413881 + 9223371130308362513 cannot be represented in type 'long'
21:03:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.241411 3346 raft_consensus.cc:1270] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Refusing update from remote peer 368a4202fe704bd8addedbf58359d23f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
21:03:20.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.242182 4029 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.242831 3241 consensus_peers.cc:487] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f -> Peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Couldn't send request to peer c8e1b47b9a32447d8758d067e7a5dd60. Status: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.242691 3255 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55616:
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.244495 4028 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 368a4202fe704bd8addedbf58359d23f. Latest consensus state: current_term: 3 leader_uuid: "368a4202fe704bd8addedbf58359d23f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.244784 4028 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: This master's current role is: FOLLOWER
21:03:20.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.245234 3255 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:03:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.248006 4028 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "368a4202fe704bd8addedbf58359d23f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.248359 4028 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [sys.catalog]: This master's current role is: FOLLOWER
21:03:20.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.258069 4037 sys_catalog.cc:455] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "368a4202fe704bd8addedbf58359d23f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } } }
21:03:20.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:20.258523 4037 sys_catalog.cc:458] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f [sys.catalog]: This master's current role is: LEADER
21:03:20.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.333782 3807 heartbeater.cc:643] Failed to heartbeat to 127.3.35.124:46649 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.523312 3949 heartbeater.cc:643] Failed to heartbeat to 127.3.35.124:46649 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:20.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.706946 3241 consensus_peers.cc:487] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f -> Peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Couldn't send request to peer c8e1b47b9a32447d8758d067e7a5dd60. Status: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:03:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:20.919150 3241 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111) [suppressed 18 similar messages]
21:03:21.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:21.193518 3241 consensus_peers.cc:487] T 00000000000000000000000000000000 P 368a4202fe704bd8addedbf58359d23f -> Peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Couldn't send request to peer c8e1b47b9a32447d8758d067e7a5dd60. Status: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:03:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f5aa58e8b08 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28
21:03:21.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0xa614e5 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/countdown_latch.h:99:38
21:03:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f5ab8e2ea32 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/promise.h:57:16
21:03:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f5ab8e1f204 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/master.cc:458:48
21:03:21.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f5ab8f78b3e in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:283:21
21:03:21.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f5ab8f7971b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:226:3
21:03:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f5ab8f7e38f in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager_service.cc:159:42
21:03:21.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f5ab2f3aaa6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/asan/src/kudu/master/txn_manager.service.cc:271:13
21:03:21.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f5ab2f3a7d5 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
21:03:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f5aac6f530e in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
21:03:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f5aac6f41ee in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3
21:03:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f5aac6fa425 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15
21:03:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f5aac6fad4f in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5
21:03:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f5aac6fab78 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
21:03:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0xa71ecc in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
21:03:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f5aa59b30bd in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:693:3
21:03:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f5aa8d966da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
21:03:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f5aa1b0671e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
21:03:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28 in
21:03:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.334990 3808 heartbeater.cc:502] Master 127.3.35.126:35753 was elected leader, sending a full tablet report...
21:03:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.349411 3950 heartbeater.cc:502] Master 127.3.35.126:35753 was elected leader, sending a full tablet report...
21:03:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:21.355412 3828 connection.cc:537] client connection to 127.3.35.126:35753 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:03:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:21.355559 3686 connection.cc:537] client connection to 127.3.35.126:35753 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:03:21.355 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.35.126:35753(127.3.35.126:35753)] handling channelInactive
21:03:21.355 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state READY due to: connection closed
21:03:21.356 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.3.35.126:35753(127.3.35.126:35753) for tablet Kudu Master: connection closed
21:03:21.359 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:21.543 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.544 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.544 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.544 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.544 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.545 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.549 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.559 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.579 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.580 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.599 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.600 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.615499 4061 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:21.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.615689 4061 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:21.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.616353 4061 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:21.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:21.617271 3322 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111) [suppressed 6 similar messages]
21:03:21.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:21.618640 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:21.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:21.618822 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:21.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.618980 3320 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:21.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.620142 4061 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:21.620 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.621 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.660 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.720 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.860 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.860 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.879753 3667 heartbeater.cc:502] Master 127.3.35.126:35753 was elected leader, sending a full tablet report...
21:03:21.880 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.881 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.930413 4061 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.930620 4061 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.931386 4061 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:21.932828 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:21.932979 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.933087 3320 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:21.933297 4061 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:21.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.939 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:21.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:21.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:21.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:21.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:21.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.000 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.060 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [44ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@214306662(state=PENDING, result=null, callback=(continuation of Deferred@421913662 after retry RPC after error@737625986) -> (continuation of Deferred@2091435391 after retry RPC after error@1084840115) -> (continuation of Deferred@1981497484 after retry RPC after error@1732362906) -> (continuation of Deferred@1568174128 after retry RPC after error@1461635052) -> (continuation of Deferred@2066604707 after retry RPC after error@303151694) -> (continuation of Deferred@217489868 after retry RPC after error@771110425) -> (continuation of Deferred@962257406 after retry RPC after error@646377297), errback=(continuation of Deferred@421913662 after retry RPC after error@737625986) -> (continuation of Deferred@2091435391 after retry RPC after error@1084840115) -> (continuation of Deferred@1981497484 after retry RPC after error@1732362906) -> (continuation of Deferred@1568174128 after retry RPC after error@1461635052) -> (continuation of Deferred@2066604707 after retry RPC after error@303151694) -> (continuation of Deferred@217489868 after retry RPC after error@771110425) -> (continuation of Deferred@962257406 after retry RPC after error@646377297)))
21:03:22.061 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [44ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@214306662(state=PENDING, result=null, callback=(continuation of Deferred@421913662 after retry RPC after error@737625986) -> (continuation of Deferred@2091435391 after retry RPC after error@1084840115) -> (continuation of Deferred@1981497484 after retry RPC after error@1732362906) -> (continuation of Deferred@1568174128 after retry RPC after error@1461635052) -> (continuation of Deferred@2066604707 after retry RPC after error@303151694) -> (continuation of Deferred@217489868 after retry RPC after error@771110425) -> (continuation of Deferred@962257406 after retry RPC after error@646377297), errback=(continuation of Deferred@421913662 after retry RPC after error@737625986) -> (continuation of Deferred@2091435391 after retry RPC after error@1084840115) -> (continuation of Deferred@1981497484 after retry RPC after error@1732362906) -> (continuation of Deferred@1568174128 after retry RPC after error@1461635052) -> (continuation of Deferred@2066604707 after retry RPC after error@303151694) -> (continuation of Deferred@217489868 after retry RPC after error@771110425) -> (continuation of Deferred@962257406 after retry RPC after error@646377297)))
21:03:22.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.079 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.099 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.101 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.179 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.180 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.180 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.259 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.259 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.259 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.260 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.260 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@460945984(state=PENDING, result=null, callback=(continuation of Deferred@2103123214 after retry RPC after error@460338410) -> (continuation of Deferred@105374507 after retry RPC after error@1446785753) -> (continuation of Deferred@1932195453 after retry RPC after error@1768377238) -> (continuation of Deferred@1342223146 after retry RPC after error@1738406153) -> (continuation of Deferred@1126195865 after retry RPC after error@281590416) -> (continuation of Deferred@1166328384 after retry RPC after error@1306082888), errback=(continuation of Deferred@2103123214 after retry RPC after error@460338410) -> (continuation of Deferred@105374507 after retry RPC after error@1446785753) -> (continuation of Deferred@1932195453 after retry RPC after error@1768377238) -> (continuation of Deferred@1342223146 after retry RPC after error@1738406153) -> (continuation of Deferred@1126195865 after retry RPC after error@281590416) -> (continuation of Deferred@1166328384 after retry RPC after error@1306082888)))
21:03:22.261 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@460945984(state=PENDING, result=null, callback=(continuation of Deferred@2103123214 after retry RPC after error@460338410) -> (continuation of Deferred@105374507 after retry RPC after error@1446785753) -> (continuation of Deferred@1932195453 after retry RPC after error@1768377238) -> (continuation of Deferred@1342223146 after retry RPC after error@1738406153) -> (continuation of Deferred@1126195865 after retry RPC after error@281590416) -> (continuation of Deferred@1166328384 after retry RPC after error@1306082888), errback=(continuation of Deferred@2103123214 after retry RPC after error@460338410) -> (continuation of Deferred@105374507 after retry RPC after error@1446785753) -> (continuation of Deferred@1932195453 after retry RPC after error@1768377238) -> (continuation of Deferred@1342223146 after retry RPC after error@1738406153) -> (continuation of Deferred@1126195865 after retry RPC after error@281590416) -> (continuation of Deferred@1166328384 after retry RPC after error@1306082888)))
21:03:22.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.279 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.279 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.279 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.280 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.339 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.340 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.340 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.400 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.400 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.400 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.459 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.460 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.460 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1999170641(state=PENDING, result=null, callback=(continuation of Deferred@459386135 after retry RPC after error@418341291) -> (continuation of Deferred@63246988 after retry RPC after error@868170945) -> (continuation of Deferred@631252660 after retry RPC after error@1176631698) -> (continuation of Deferred@1834734248 after retry RPC after error@667464438) -> (continuation of Deferred@58307901 after retry RPC after error@1256006057) -> (continuation of Deferred@2040554696 after retry RPC after error@454899208) -> (continuation of Deferred@1434691595 after retry RPC after error@1987054459), errback=(continuation of Deferred@459386135 after retry RPC after error@418341291) -> (continuation of Deferred@63246988 after retry RPC after error@868170945) -> (continuation of Deferred@631252660 after retry RPC after error@1176631698) -> (continuation of Deferred@1834734248 after retry RPC after error@667464438) -> (continuation of Deferred@58307901 after retry RPC after error@1256006057) -> (continuation of Deferred@2040554696 after retry RPC after error@454899208) -> (continuation of Deferred@1434691595 after retry RPC after error@1987054459)))
21:03:22.461 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1999170641(state=PENDING, result=null, callback=(continuation of Deferred@459386135 after retry RPC after error@418341291) -> (continuation of Deferred@63246988 after retry RPC after error@868170945) -> (continuation of Deferred@631252660 after retry RPC after error@1176631698) -> (continuation of Deferred@1834734248 after retry RPC after error@667464438) -> (continuation of Deferred@58307901 after retry RPC after error@1256006057) -> (continuation of Deferred@2040554696 after retry RPC after error@454899208) -> (continuation of Deferred@1434691595 after retry RPC after error@1987054459), errback=(continuation of Deferred@459386135 after retry RPC after error@418341291) -> (continuation of Deferred@63246988 after retry RPC after error@868170945) -> (continuation of Deferred@631252660 after retry RPC after error@1176631698) -> (continuation of Deferred@1834734248 after retry RPC after error@667464438) -> (continuation of Deferred@58307901 after retry RPC after error@1256006057) -> (continuation of Deferred@2040554696 after retry RPC after error@454899208) -> (continuation of Deferred@1434691595 after retry RPC after error@1987054459)))
21:03:22.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.480 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.480 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.480 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.480 [INFO - pool-38-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.481 [WARN - pool-38-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.501 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:22.503664 3548 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.170s from last keepalive heartbeat (effective timeout is 1.000s)
21:03:22.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:22.505091 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:03:22.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.519 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.539 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.539 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.539 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.541 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:22.557091 4081 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:22.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:22.557511 4081 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:22.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.561 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:22.563230 4081 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:22.563975 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:22.564311 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:22.564448 3322 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:22.564702 4081 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:22.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.599 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.599 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.600 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.639 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.640 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.641 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.699 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.700 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.700 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.700 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.700 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.701 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [4ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [164ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1229399402(state=PENDING, result=null, callback=(continuation of Deferred@734106607 after retry RPC after error@80575798) -> (continuation of Deferred@1178078392 after retry RPC after error@908336480) -> (continuation of Deferred@299740832 after retry RPC after error@2123781472) -> (continuation of Deferred@2018368454 after retry RPC after error@1855194802) -> (continuation of Deferred@306187630 after retry RPC after error@1580295955) -> (continuation of Deferred@1219761400 after retry RPC after error@115026634) -> (continuation of Deferred@350383106 after retry RPC after error@1094617202), errback=(continuation of Deferred@734106607 after retry RPC after error@80575798) -> (continuation of Deferred@1178078392 after retry RPC after error@908336480) -> (continuation of Deferred@299740832 after retry RPC after error@2123781472) -> (continuation of Deferred@2018368454 after retry RPC after error@1855194802) -> (continuation of Deferred@306187630 after retry RPC after error@1580295955) -> (continuation of Deferred@1219761400 after retry RPC after error@115026634) -> (continuation of Deferred@350383106 after retry RPC after error@1094617202)))
21:03:22.701 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [4ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [164ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1229399402(state=PENDING, result=null, callback=(continuation of Deferred@734106607 after retry RPC after error@80575798) -> (continuation of Deferred@1178078392 after retry RPC after error@908336480) -> (continuation of Deferred@299740832 after retry RPC after error@2123781472) -> (continuation of Deferred@2018368454 after retry RPC after error@1855194802) -> (continuation of Deferred@306187630 after retry RPC after error@1580295955) -> (continuation of Deferred@1219761400 after retry RPC after error@115026634) -> (continuation of Deferred@350383106 after retry RPC after error@1094617202), errback=(continuation of Deferred@734106607 after retry RPC after error@80575798) -> (continuation of Deferred@1178078392 after retry RPC after error@908336480) -> (continuation of Deferred@299740832 after retry RPC after error@2123781472) -> (continuation of Deferred@2018368454 after retry RPC after error@1855194802) -> (continuation of Deferred@306187630 after retry RPC after error@1580295955) -> (continuation of Deferred@1219761400 after retry RPC after error@115026634) -> (continuation of Deferred@350383106 after retry RPC after error@1094617202)))
21:03:22.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.720 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.739 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.799 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.859 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@608512522(state=PENDING, result=null, callback=(continuation of Deferred@784853003 after retry RPC after error@914742721) -> (continuation of Deferred@738454379 after retry RPC after error@563843483) -> (continuation of Deferred@814588435 after retry RPC after error@687737366) -> (continuation of Deferred@454423761 after retry RPC after error@1728031073) -> (continuation of Deferred@1002377879 after retry RPC after error@1665728844) -> (continuation of Deferred@1464779400 after retry RPC after error@1289866657) -> (continuation of Deferred@252269824 after retry RPC after error@610367180), errback=(continuation of Deferred@784853003 after retry RPC after error@914742721) -> (continuation of Deferred@738454379 after retry RPC after error@563843483) -> (continuation of Deferred@814588435 after retry RPC after error@687737366) -> (continuation of Deferred@454423761 after retry RPC after error@1728031073) -> (continuation of Deferred@1002377879 after retry RPC after error@1665728844) -> (continuation of Deferred@1464779400 after retry RPC after error@1289866657) -> (continuation of Deferred@252269824 after retry RPC after error@610367180)))
21:03:22.880 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@608512522(state=PENDING, result=null, callback=(continuation of Deferred@784853003 after retry RPC after error@914742721) -> (continuation of Deferred@738454379 after retry RPC after error@563843483) -> (continuation of Deferred@814588435 after retry RPC after error@687737366) -> (continuation of Deferred@454423761 after retry RPC after error@1728031073) -> (continuation of Deferred@1002377879 after retry RPC after error@1665728844) -> (continuation of Deferred@1464779400 after retry RPC after error@1289866657) -> (continuation of Deferred@252269824 after retry RPC after error@610367180), errback=(continuation of Deferred@784853003 after retry RPC after error@914742721) -> (continuation of Deferred@738454379 after retry RPC after error@563843483) -> (continuation of Deferred@814588435 after retry RPC after error@687737366) -> (continuation of Deferred@454423761 after retry RPC after error@1728031073) -> (continuation of Deferred@1002377879 after retry RPC after error@1665728844) -> (continuation of Deferred@1464779400 after retry RPC after error@1289866657) -> (continuation of Deferred@252269824 after retry RPC after error@610367180)))
21:03:22.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.939 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.939 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.939 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.940 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.959 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:22.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:22.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:22.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:22.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:22.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.019 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.020 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.020 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.022 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.022 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.041 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.059 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.059 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.080 [INFO - pool-38-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.080 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.081 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.120 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.120 [INFO - pool-38-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.120 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.120 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.122 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.123 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=225), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [126ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [224ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [225ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1698800026(state=PENDING, result=null, callback=(continuation of Deferred@1026465430 after retry RPC after error@940229712) -> (continuation of Deferred@588040357 after retry RPC after error@1269424175) -> (continuation of Deferred@45751778 after retry RPC after error@1820090785) -> (continuation of Deferred@703645035 after retry RPC after error@761061602) -> (continuation of Deferred@2136714662 after retry RPC after error@1889905285) -> (continuation of Deferred@1599201895 after retry RPC after error@356532895) -> (continuation of Deferred@928312076 after retry RPC after error@1794678497) -> (continuation of Deferred@980028163 after retry RPC after error@757723755), errback=(continuation of Deferred@1026465430 after retry RPC after error@940229712) -> (continuation of Deferred@588040357 after retry RPC after error@1269424175) -> (continuation of Deferred@45751778 after retry RPC after error@1820090785) -> (continuation of Deferred@703645035 after retry RPC after error@761061602) -> (continuation of Deferred@2136714662 after retry RPC after error@1889905285) -> (continuation of Deferred@1599201895 after retry RPC after error@356532895) -> (continuation of Deferred@928312076 after retry RPC after error@1794678497) -> (continuation of Deferred@980028163 after retry RPC after error@757723755)))
21:03:23.123 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=225), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [126ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [224ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [225ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1698800026(state=PENDING, result=null, callback=(continuation of Deferred@1026465430 after retry RPC after error@940229712) -> (continuation of Deferred@588040357 after retry RPC after error@1269424175) -> (continuation of Deferred@45751778 after retry RPC after error@1820090785) -> (continuation of Deferred@703645035 after retry RPC after error@761061602) -> (continuation of Deferred@2136714662 after retry RPC after error@1889905285) -> (continuation of Deferred@1599201895 after retry RPC after error@356532895) -> (continuation of Deferred@928312076 after retry RPC after error@1794678497) -> (continuation of Deferred@980028163 after retry RPC after error@757723755), errback=(continuation of Deferred@1026465430 after retry RPC after error@940229712) -> (continuation of Deferred@588040357 after retry RPC after error@1269424175) -> (continuation of Deferred@45751778 after retry RPC after error@1820090785) -> (continuation of Deferred@703645035 after retry RPC after error@761061602) -> (continuation of Deferred@2136714662 after retry RPC after error@1889905285) -> (continuation of Deferred@1599201895 after retry RPC after error@356532895) -> (continuation of Deferred@928312076 after retry RPC after error@1794678497) -> (continuation of Deferred@980028163 after retry RPC after error@757723755)))
21:03:23.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.140 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.140 [INFO - pool-38-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.140 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.140 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.140 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.160 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.160 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.161 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.179 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.179 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.179 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.199 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:23.211087 4087 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:23.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:23.211309 4087 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:23.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:23.213284 4087 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:23.214565 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:23.214779 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:23.214910 3322 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:23.215183 4087 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:23.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.220 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.220 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.221 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.259 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.259 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.259 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.260 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.397 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [25ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1486368453(state=PENDING, result=null, callback=(continuation of Deferred@339897882 after retry RPC after error@59800488) -> (continuation of Deferred@1590102098 after retry RPC after error@1194388297) -> (continuation of Deferred@1082486539 after retry RPC after error@1078460808) -> (continuation of Deferred@648746059 after retry RPC after error@1711397236) -> (continuation of Deferred@219615112 after retry RPC after error@223546339) -> (continuation of Deferred@1931759438 after retry RPC after error@453440039) -> (continuation of Deferred@785958272 after retry RPC after error@994172843), errback=(continuation of Deferred@339897882 after retry RPC after error@59800488) -> (continuation of Deferred@1590102098 after retry RPC after error@1194388297) -> (continuation of Deferred@1082486539 after retry RPC after error@1078460808) -> (continuation of Deferred@648746059 after retry RPC after error@1711397236) -> (continuation of Deferred@219615112 after retry RPC after error@223546339) -> (continuation of Deferred@1931759438 after retry RPC after error@453440039) -> (continuation of Deferred@785958272 after retry RPC after error@994172843)))
21:03:23.398 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:23.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.419 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.419 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.459 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.460 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.519 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.677 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@56815341(state=PENDING, result=null, callback=(continuation of Deferred@265857064 after retry RPC after error@1694161269) -> (continuation of Deferred@1865597311 after retry RPC after error@2066964466) -> (continuation of Deferred@836772180 after retry RPC after error@65234796) -> (continuation of Deferred@2049140566 after retry RPC after error@453933871) -> (continuation of Deferred@735318939 after retry RPC after error@857000330) -> (continuation of Deferred@1391198569 after retry RPC after error@978571032) -> (continuation of Deferred@831544066 after retry RPC after error@610114595), errback=(continuation of Deferred@265857064 after retry RPC after error@1694161269) -> (continuation of Deferred@1865597311 after retry RPC after error@2066964466) -> (continuation of Deferred@836772180 after retry RPC after error@65234796) -> (continuation of Deferred@2049140566 after retry RPC after error@453933871) -> (continuation of Deferred@735318939 after retry RPC after error@857000330) -> (continuation of Deferred@1391198569 after retry RPC after error@978571032) -> (continuation of Deferred@831544066 after retry RPC after error@610114595)))
21:03:23.678 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:23.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.739 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.760 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.780 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.819 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.939 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.939 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.940 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@943335552(state=PENDING, result=null, callback=(continuation of Deferred@1834201511 after retry RPC after error@396151211) -> (continuation of Deferred@590202051 after retry RPC after error@1245353486) -> (continuation of Deferred@982829159 after retry RPC after error@1270124303) -> (continuation of Deferred@116539750 after retry RPC after error@417742160) -> (continuation of Deferred@78453211 after retry RPC after error@1796092598) -> (continuation of Deferred@2145626048 after retry RPC after error@1455819217) -> (continuation of Deferred@977907987 after retry RPC after error@541634414), errback=(continuation of Deferred@1834201511 after retry RPC after error@396151211) -> (continuation of Deferred@590202051 after retry RPC after error@1245353486) -> (continuation of Deferred@982829159 after retry RPC after error@1270124303) -> (continuation of Deferred@116539750 after retry RPC after error@417742160) -> (continuation of Deferred@78453211 after retry RPC after error@1796092598) -> (continuation of Deferred@2145626048 after retry RPC after error@1455819217) -> (continuation of Deferred@977907987 after retry RPC after error@541634414)))
21:03:23.940 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@943335552(state=PENDING, result=null, callback=(continuation of Deferred@1834201511 after retry RPC after error@396151211) -> (continuation of Deferred@590202051 after retry RPC after error@1245353486) -> (continuation of Deferred@982829159 after retry RPC after error@1270124303) -> (continuation of Deferred@116539750 after retry RPC after error@417742160) -> (continuation of Deferred@78453211 after retry RPC after error@1796092598) -> (continuation of Deferred@2145626048 after retry RPC after error@1455819217) -> (continuation of Deferred@977907987 after retry RPC after error@541634414), errback=(continuation of Deferred@1834201511 after retry RPC after error@396151211) -> (continuation of Deferred@590202051 after retry RPC after error@1245353486) -> (continuation of Deferred@982829159 after retry RPC after error@1270124303) -> (continuation of Deferred@116539750 after retry RPC after error@417742160) -> (continuation of Deferred@78453211 after retry RPC after error@1796092598) -> (continuation of Deferred@2145626048 after retry RPC after error@1455819217) -> (continuation of Deferred@977907987 after retry RPC after error@541634414)))
21:03:23.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.959 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:23.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:23.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:23.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:23.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:23.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.020 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.059 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.059 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.060 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:24.111109 4095 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:24.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:24.111330 4095 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:24.112114 4095 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:24.114097 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:24.114257 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:24.114444 3320 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:24.114712 4095 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:24.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.199 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.200 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.201 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1954156915(state=PENDING, result=null, callback=(continuation of Deferred@1867985555 after retry RPC after error@1248856835) -> (continuation of Deferred@1780027351 after retry RPC after error@1322023534) -> (continuation of Deferred@416881294 after retry RPC after error@1379611269) -> (continuation of Deferred@1895753890 after retry RPC after error@1573708761) -> (continuation of Deferred@227647364 after retry RPC after error@112540585) -> (continuation of Deferred@17397797 after retry RPC after error@996246006) -> (continuation of Deferred@973948065 after retry RPC after error@365578298), errback=(continuation of Deferred@1867985555 after retry RPC after error@1248856835) -> (continuation of Deferred@1780027351 after retry RPC after error@1322023534) -> (continuation of Deferred@416881294 after retry RPC after error@1379611269) -> (continuation of Deferred@1895753890 after retry RPC after error@1573708761) -> (continuation of Deferred@227647364 after retry RPC after error@112540585) -> (continuation of Deferred@17397797 after retry RPC after error@996246006) -> (continuation of Deferred@973948065 after retry RPC after error@365578298)))
21:03:24.202 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1954156915(state=PENDING, result=null, callback=(continuation of Deferred@1867985555 after retry RPC after error@1248856835) -> (continuation of Deferred@1780027351 after retry RPC after error@1322023534) -> (continuation of Deferred@416881294 after retry RPC after error@1379611269) -> (continuation of Deferred@1895753890 after retry RPC after error@1573708761) -> (continuation of Deferred@227647364 after retry RPC after error@112540585) -> (continuation of Deferred@17397797 after retry RPC after error@996246006) -> (continuation of Deferred@973948065 after retry RPC after error@365578298), errback=(continuation of Deferred@1867985555 after retry RPC after error@1248856835) -> (continuation of Deferred@1780027351 after retry RPC after error@1322023534) -> (continuation of Deferred@416881294 after retry RPC after error@1379611269) -> (continuation of Deferred@1895753890 after retry RPC after error@1573708761) -> (continuation of Deferred@227647364 after retry RPC after error@112540585) -> (continuation of Deferred@17397797 after retry RPC after error@996246006) -> (continuation of Deferred@973948065 after retry RPC after error@365578298)))
21:03:24.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.279 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.279 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.279 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.419 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.459 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.459 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.460 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.461 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@788649420(state=PENDING, result=null, callback=(continuation of Deferred@1071760455 after retry RPC after error@431732234) -> (continuation of Deferred@2043902283 after retry RPC after error@1490910216) -> (continuation of Deferred@1764706472 after retry RPC after error@769697889) -> (continuation of Deferred@1335592375 after retry RPC after error@2081791344) -> (continuation of Deferred@1869410386 after retry RPC after error@1197997694) -> (continuation of Deferred@1422028056 after retry RPC after error@82613485) -> (continuation of Deferred@201988674 after retry RPC after error@1952874931) -> (continuation of Deferred@856397962 after retry RPC after error@1394616826), errback=(continuation of Deferred@1071760455 after retry RPC after error@431732234) -> (continuation of Deferred@2043902283 after retry RPC after error@1490910216) -> (continuation of Deferred@1764706472 after retry RPC after error@769697889) -> (continuation of Deferred@1335592375 after retry RPC after error@2081791344) -> (continuation of Deferred@1869410386 after retry RPC after error@1197997694) -> (continuation of Deferred@1422028056 after retry RPC after error@82613485) -> (continuation of Deferred@201988674 after retry RPC after error@1952874931) -> (continuation of Deferred@856397962 after retry RPC after error@1394616826)))
21:03:24.462 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@788649420(state=PENDING, result=null, callback=(continuation of Deferred@1071760455 after retry RPC after error@431732234) -> (continuation of Deferred@2043902283 after retry RPC after error@1490910216) -> (continuation of Deferred@1764706472 after retry RPC after error@769697889) -> (continuation of Deferred@1335592375 after retry RPC after error@2081791344) -> (continuation of Deferred@1869410386 after retry RPC after error@1197997694) -> (continuation of Deferred@1422028056 after retry RPC after error@82613485) -> (continuation of Deferred@201988674 after retry RPC after error@1952874931) -> (continuation of Deferred@856397962 after retry RPC after error@1394616826), errback=(continuation of Deferred@1071760455 after retry RPC after error@431732234) -> (continuation of Deferred@2043902283 after retry RPC after error@1490910216) -> (continuation of Deferred@1764706472 after retry RPC after error@769697889) -> (continuation of Deferred@1335592375 after retry RPC after error@2081791344) -> (continuation of Deferred@1869410386 after retry RPC after error@1197997694) -> (continuation of Deferred@1422028056 after retry RPC after error@82613485) -> (continuation of Deferred@201988674 after retry RPC after error@1952874931) -> (continuation of Deferred@856397962 after retry RPC after error@1394616826)))
21:03:24.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.480 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.480 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.481 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.639 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.640 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.700 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1116351837(state=PENDING, result=null, callback=(continuation of Deferred@1193679172 after retry RPC after error@3171833) -> (continuation of Deferred@142578982 after retry RPC after error@462201265) -> (continuation of Deferred@1813473213 after retry RPC after error@1863043179) -> (continuation of Deferred@464102921 after retry RPC after error@1708531172) -> (continuation of Deferred@850113644 after retry RPC after error@55732930) -> (continuation of Deferred@1427967311 after retry RPC after error@15794997) -> (continuation of Deferred@1872154515 after retry RPC after error@2139270361), errback=(continuation of Deferred@1193679172 after retry RPC after error@3171833) -> (continuation of Deferred@142578982 after retry RPC after error@462201265) -> (continuation of Deferred@1813473213 after retry RPC after error@1863043179) -> (continuation of Deferred@464102921 after retry RPC after error@1708531172) -> (continuation of Deferred@850113644 after retry RPC after error@55732930) -> (continuation of Deferred@1427967311 after retry RPC after error@15794997) -> (continuation of Deferred@1872154515 after retry RPC after error@2139270361)))
21:03:24.700 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1116351837(state=PENDING, result=null, callback=(continuation of Deferred@1193679172 after retry RPC after error@3171833) -> (continuation of Deferred@142578982 after retry RPC after error@462201265) -> (continuation of Deferred@1813473213 after retry RPC after error@1863043179) -> (continuation of Deferred@464102921 after retry RPC after error@1708531172) -> (continuation of Deferred@850113644 after retry RPC after error@55732930) -> (continuation of Deferred@1427967311 after retry RPC after error@15794997) -> (continuation of Deferred@1872154515 after retry RPC after error@2139270361), errback=(continuation of Deferred@1193679172 after retry RPC after error@3171833) -> (continuation of Deferred@142578982 after retry RPC after error@462201265) -> (continuation of Deferred@1813473213 after retry RPC after error@1863043179) -> (continuation of Deferred@464102921 after retry RPC after error@1708531172) -> (continuation of Deferred@850113644 after retry RPC after error@55732930) -> (continuation of Deferred@1427967311 after retry RPC after error@15794997) -> (continuation of Deferred@1872154515 after retry RPC after error@2139270361)))
21:03:24.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.900 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@924143145(state=PENDING, result=null, callback=(continuation of Deferred@1971382092 after retry RPC after error@49526124) -> (continuation of Deferred@1635477061 after retry RPC after error@2088900677) -> (continuation of Deferred@239741470 after retry RPC after error@294495854) -> (continuation of Deferred@738112340 after retry RPC after error@1204999774) -> (continuation of Deferred@412552800 after retry RPC after error@1691710922) -> (continuation of Deferred@988745239 after retry RPC after error@1809731378), errback=(continuation of Deferred@1971382092 after retry RPC after error@49526124) -> (continuation of Deferred@1635477061 after retry RPC after error@2088900677) -> (continuation of Deferred@239741470 after retry RPC after error@294495854) -> (continuation of Deferred@738112340 after retry RPC after error@1204999774) -> (continuation of Deferred@412552800 after retry RPC after error@1691710922) -> (continuation of Deferred@988745239 after retry RPC after error@1809731378)))
21:03:24.900 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@924143145(state=PENDING, result=null, callback=(continuation of Deferred@1971382092 after retry RPC after error@49526124) -> (continuation of Deferred@1635477061 after retry RPC after error@2088900677) -> (continuation of Deferred@239741470 after retry RPC after error@294495854) -> (continuation of Deferred@738112340 after retry RPC after error@1204999774) -> (continuation of Deferred@412552800 after retry RPC after error@1691710922) -> (continuation of Deferred@988745239 after retry RPC after error@1809731378), errback=(continuation of Deferred@1971382092 after retry RPC after error@49526124) -> (continuation of Deferred@1635477061 after retry RPC after error@2088900677) -> (continuation of Deferred@239741470 after retry RPC after error@294495854) -> (continuation of Deferred@738112340 after retry RPC after error@1204999774) -> (continuation of Deferred@412552800 after retry RPC after error@1691710922) -> (continuation of Deferred@988745239 after retry RPC after error@1809731378)))
21:03:24.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:24.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:24.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:24.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:24.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:24.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.079 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.081 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@498183294(state=PENDING, result=null, callback=(continuation of Deferred@839638786 after retry RPC after error@334323243) -> (continuation of Deferred@108055274 after retry RPC after error@380785670) -> (continuation of Deferred@2058189341 after retry RPC after error@55078757) -> (continuation of Deferred@57655932 after retry RPC after error@125412467) -> (continuation of Deferred@370498323 after retry RPC after error@1457929715) -> (continuation of Deferred@728197489 after retry RPC after error@1635195474), errback=(continuation of Deferred@839638786 after retry RPC after error@334323243) -> (continuation of Deferred@108055274 after retry RPC after error@380785670) -> (continuation of Deferred@2058189341 after retry RPC after error@55078757) -> (continuation of Deferred@57655932 after retry RPC after error@125412467) -> (continuation of Deferred@370498323 after retry RPC after error@1457929715) -> (continuation of Deferred@728197489 after retry RPC after error@1635195474)))
21:03:25.081 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@498183294(state=PENDING, result=null, callback=(continuation of Deferred@839638786 after retry RPC after error@334323243) -> (continuation of Deferred@108055274 after retry RPC after error@380785670) -> (continuation of Deferred@2058189341 after retry RPC after error@55078757) -> (continuation of Deferred@57655932 after retry RPC after error@125412467) -> (continuation of Deferred@370498323 after retry RPC after error@1457929715) -> (continuation of Deferred@728197489 after retry RPC after error@1635195474), errback=(continuation of Deferred@839638786 after retry RPC after error@334323243) -> (continuation of Deferred@108055274 after retry RPC after error@380785670) -> (continuation of Deferred@2058189341 after retry RPC after error@55078757) -> (continuation of Deferred@57655932 after retry RPC after error@125412467) -> (continuation of Deferred@370498323 after retry RPC after error@1457929715) -> (continuation of Deferred@728197489 after retry RPC after error@1635195474)))
21:03:25.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.099 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.100 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.179 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.259 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.260 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@922971270(state=PENDING, result=null, callback=(continuation of Deferred@170794432 after retry RPC after error@2057095196) -> (continuation of Deferred@1789606303 after retry RPC after error@1180700432) -> (continuation of Deferred@1020971521 after retry RPC after error@538585518) -> (continuation of Deferred@323918835 after retry RPC after error@1435508229) -> (continuation of Deferred@1064328308 after retry RPC after error@1314280813) -> (continuation of Deferred@690015831 after retry RPC after error@2090175808) -> (continuation of Deferred@1102835034 after retry RPC after error@1035613501), errback=(continuation of Deferred@170794432 after retry RPC after error@2057095196) -> (continuation of Deferred@1789606303 after retry RPC after error@1180700432) -> (continuation of Deferred@1020971521 after retry RPC after error@538585518) -> (continuation of Deferred@323918835 after retry RPC after error@1435508229) -> (continuation of Deferred@1064328308 after retry RPC after error@1314280813) -> (continuation of Deferred@690015831 after retry RPC after error@2090175808) -> (continuation of Deferred@1102835034 after retry RPC after error@1035613501)))
21:03:25.261 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@922971270(state=PENDING, result=null, callback=(continuation of Deferred@170794432 after retry RPC after error@2057095196) -> (continuation of Deferred@1789606303 after retry RPC after error@1180700432) -> (continuation of Deferred@1020971521 after retry RPC after error@538585518) -> (continuation of Deferred@323918835 after retry RPC after error@1435508229) -> (continuation of Deferred@1064328308 after retry RPC after error@1314280813) -> (continuation of Deferred@690015831 after retry RPC after error@2090175808) -> (continuation of Deferred@1102835034 after retry RPC after error@1035613501), errback=(continuation of Deferred@170794432 after retry RPC after error@2057095196) -> (continuation of Deferred@1789606303 after retry RPC after error@1180700432) -> (continuation of Deferred@1020971521 after retry RPC after error@538585518) -> (continuation of Deferred@323918835 after retry RPC after error@1435508229) -> (continuation of Deferred@1064328308 after retry RPC after error@1314280813) -> (continuation of Deferred@690015831 after retry RPC after error@2090175808) -> (continuation of Deferred@1102835034 after retry RPC after error@1035613501)))
21:03:25.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.279 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.300 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.300 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.301 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.537 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@3302768(state=PENDING, result=null, callback=(continuation of Deferred@1599206593 after retry RPC after error@596045314) -> (continuation of Deferred@957512695 after retry RPC after error@714637376) -> (continuation of Deferred@2050989614 after retry RPC after error@1860012437) -> (continuation of Deferred@1153079930 after retry RPC after error@709789741) -> (continuation of Deferred@747023780 after retry RPC after error@773134464) -> (continuation of Deferred@1055669874 after retry RPC after error@1623049826) -> (continuation of Deferred@2035268700 after retry RPC after error@2132306669) -> (continuation of Deferred@551081170 after retry RPC after error@777558141), errback=(continuation of Deferred@1599206593 after retry RPC after error@596045314) -> (continuation of Deferred@957512695 after retry RPC after error@714637376) -> (continuation of Deferred@2050989614 after retry RPC after error@1860012437) -> (continuation of Deferred@1153079930 after retry RPC after error@709789741) -> (continuation of Deferred@747023780 after retry RPC after error@773134464) -> (continuation of Deferred@1055669874 after retry RPC after error@1623049826) -> (continuation of Deferred@2035268700 after retry RPC after error@2132306669) -> (continuation of Deferred@551081170 after retry RPC after error@777558141)))
21:03:25.538 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:25.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.579 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.579 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.580 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.620 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.640 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.760 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1039410359(state=PENDING, result=null, callback=(continuation of Deferred@1415094165 after retry RPC after error@902812203) -> (continuation of Deferred@370915972 after retry RPC after error@173466988) -> (continuation of Deferred@451049348 after retry RPC after error@1375182969) -> (continuation of Deferred@443809625 after retry RPC after error@666043646) -> (continuation of Deferred@786700763 after retry RPC after error@1123848214) -> (continuation of Deferred@1206920743 after retry RPC after error@2019253971) -> (continuation of Deferred@1996466882 after retry RPC after error@1344665643), errback=(continuation of Deferred@1415094165 after retry RPC after error@902812203) -> (continuation of Deferred@370915972 after retry RPC after error@173466988) -> (continuation of Deferred@451049348 after retry RPC after error@1375182969) -> (continuation of Deferred@443809625 after retry RPC after error@666043646) -> (continuation of Deferred@786700763 after retry RPC after error@1123848214) -> (continuation of Deferred@1206920743 after retry RPC after error@2019253971) -> (continuation of Deferred@1996466882 after retry RPC after error@1344665643)))
21:03:25.761 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1039410359(state=PENDING, result=null, callback=(continuation of Deferred@1415094165 after retry RPC after error@902812203) -> (continuation of Deferred@370915972 after retry RPC after error@173466988) -> (continuation of Deferred@451049348 after retry RPC after error@1375182969) -> (continuation of Deferred@443809625 after retry RPC after error@666043646) -> (continuation of Deferred@786700763 after retry RPC after error@1123848214) -> (continuation of Deferred@1206920743 after retry RPC after error@2019253971) -> (continuation of Deferred@1996466882 after retry RPC after error@1344665643), errback=(continuation of Deferred@1415094165 after retry RPC after error@902812203) -> (continuation of Deferred@370915972 after retry RPC after error@173466988) -> (continuation of Deferred@451049348 after retry RPC after error@1375182969) -> (continuation of Deferred@443809625 after retry RPC after error@666043646) -> (continuation of Deferred@786700763 after retry RPC after error@1123848214) -> (continuation of Deferred@1206920743 after retry RPC after error@2019253971) -> (continuation of Deferred@1996466882 after retry RPC after error@1344665643)))
21:03:25.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.799 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.919 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.920 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1245411044(state=PENDING, result=null, callback=(continuation of Deferred@1233829406 after retry RPC after error@170083828) -> (continuation of Deferred@1591625988 after retry RPC after error@1338218605) -> (continuation of Deferred@1403368901 after retry RPC after error@1136215712) -> (continuation of Deferred@1870897594 after retry RPC after error@1011899607) -> (continuation of Deferred@1118166277 after retry RPC after error@1260396471) -> (continuation of Deferred@970778904 after retry RPC after error@122993680) -> (continuation of Deferred@1802830858 after retry RPC after error@551777140), errback=(continuation of Deferred@1233829406 after retry RPC after error@170083828) -> (continuation of Deferred@1591625988 after retry RPC after error@1338218605) -> (continuation of Deferred@1403368901 after retry RPC after error@1136215712) -> (continuation of Deferred@1870897594 after retry RPC after error@1011899607) -> (continuation of Deferred@1118166277 after retry RPC after error@1260396471) -> (continuation of Deferred@970778904 after retry RPC after error@122993680) -> (continuation of Deferred@1802830858 after retry RPC after error@551777140)))
21:03:25.921 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1245411044(state=PENDING, result=null, callback=(continuation of Deferred@1233829406 after retry RPC after error@170083828) -> (continuation of Deferred@1591625988 after retry RPC after error@1338218605) -> (continuation of Deferred@1403368901 after retry RPC after error@1136215712) -> (continuation of Deferred@1870897594 after retry RPC after error@1011899607) -> (continuation of Deferred@1118166277 after retry RPC after error@1260396471) -> (continuation of Deferred@970778904 after retry RPC after error@122993680) -> (continuation of Deferred@1802830858 after retry RPC after error@551777140), errback=(continuation of Deferred@1233829406 after retry RPC after error@170083828) -> (continuation of Deferred@1591625988 after retry RPC after error@1338218605) -> (continuation of Deferred@1403368901 after retry RPC after error@1136215712) -> (continuation of Deferred@1870897594 after retry RPC after error@1011899607) -> (continuation of Deferred@1118166277 after retry RPC after error@1260396471) -> (continuation of Deferred@970778904 after retry RPC after error@122993680) -> (continuation of Deferred@1802830858 after retry RPC after error@551777140)))
21:03:25.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.939 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.939 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.939 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.940 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.959 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:25.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:25.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:25.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:25.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:25.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.019 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.059 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.059 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.060 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.160 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@431717119(state=PENDING, result=null, callback=(continuation of Deferred@1878920349 after retry RPC after error@1093970318) -> (continuation of Deferred@1537071204 after retry RPC after error@431177066) -> (continuation of Deferred@112607269 after retry RPC after error@1646981540) -> (continuation of Deferred@1472289661 after retry RPC after error@1461286018) -> (continuation of Deferred@1671078188 after retry RPC after error@1443420970) -> (continuation of Deferred@198146085 after retry RPC after error@401518778) -> (continuation of Deferred@567009450 after retry RPC after error@822783538), errback=(continuation of Deferred@1878920349 after retry RPC after error@1093970318) -> (continuation of Deferred@1537071204 after retry RPC after error@431177066) -> (continuation of Deferred@112607269 after retry RPC after error@1646981540) -> (continuation of Deferred@1472289661 after retry RPC after error@1461286018) -> (continuation of Deferred@1671078188 after retry RPC after error@1443420970) -> (continuation of Deferred@198146085 after retry RPC after error@401518778) -> (continuation of Deferred@567009450 after retry RPC after error@822783538)))
21:03:26.160 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@431717119(state=PENDING, result=null, callback=(continuation of Deferred@1878920349 after retry RPC after error@1093970318) -> (continuation of Deferred@1537071204 after retry RPC after error@431177066) -> (continuation of Deferred@112607269 after retry RPC after error@1646981540) -> (continuation of Deferred@1472289661 after retry RPC after error@1461286018) -> (continuation of Deferred@1671078188 after retry RPC after error@1443420970) -> (continuation of Deferred@198146085 after retry RPC after error@401518778) -> (continuation of Deferred@567009450 after retry RPC after error@822783538), errback=(continuation of Deferred@1878920349 after retry RPC after error@1093970318) -> (continuation of Deferred@1537071204 after retry RPC after error@431177066) -> (continuation of Deferred@112607269 after retry RPC after error@1646981540) -> (continuation of Deferred@1472289661 after retry RPC after error@1461286018) -> (continuation of Deferred@1671078188 after retry RPC after error@1443420970) -> (continuation of Deferred@198146085 after retry RPC after error@401518778) -> (continuation of Deferred@567009450 after retry RPC after error@822783538)))
21:03:26.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.320 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.320 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.320 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1256571253(state=PENDING, result=null, callback=(continuation of Deferred@1613177137 after retry RPC after error@977635741) -> (continuation of Deferred@1942069579 after retry RPC after error@399194159) -> (continuation of Deferred@1320992078 after retry RPC after error@1487133797) -> (continuation of Deferred@647423427 after retry RPC after error@1758641022) -> (continuation of Deferred@1127169511 after retry RPC after error@86606099) -> (continuation of Deferred@343983240 after retry RPC after error@1777632301), errback=(continuation of Deferred@1613177137 after retry RPC after error@977635741) -> (continuation of Deferred@1942069579 after retry RPC after error@399194159) -> (continuation of Deferred@1320992078 after retry RPC after error@1487133797) -> (continuation of Deferred@647423427 after retry RPC after error@1758641022) -> (continuation of Deferred@1127169511 after retry RPC after error@86606099) -> (continuation of Deferred@343983240 after retry RPC after error@1777632301)))
21:03:26.320 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1256571253(state=PENDING, result=null, callback=(continuation of Deferred@1613177137 after retry RPC after error@977635741) -> (continuation of Deferred@1942069579 after retry RPC after error@399194159) -> (continuation of Deferred@1320992078 after retry RPC after error@1487133797) -> (continuation of Deferred@647423427 after retry RPC after error@1758641022) -> (continuation of Deferred@1127169511 after retry RPC after error@86606099) -> (continuation of Deferred@343983240 after retry RPC after error@1777632301), errback=(continuation of Deferred@1613177137 after retry RPC after error@977635741) -> (continuation of Deferred@1942069579 after retry RPC after error@399194159) -> (continuation of Deferred@1320992078 after retry RPC after error@1487133797) -> (continuation of Deferred@647423427 after retry RPC after error@1758641022) -> (continuation of Deferred@1127169511 after retry RPC after error@86606099) -> (continuation of Deferred@343983240 after retry RPC after error@1777632301)))
21:03:26.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:26.372357 4109 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:26.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:26.372555 4109 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:26.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:26.373132 4109 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:26.375100 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:26.375276 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:26.375385 3322 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:26.375624 4109 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:26.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.520 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1706758849(state=PENDING, result=null, callback=(continuation of Deferred@1792783073 after retry RPC after error@1807620034) -> (continuation of Deferred@798263284 after retry RPC after error@302294493) -> (continuation of Deferred@629100998 after retry RPC after error@1046762156) -> (continuation of Deferred@902093235 after retry RPC after error@47823003) -> (continuation of Deferred@1576478823 after retry RPC after error@1092284487) -> (continuation of Deferred@1375772822 after retry RPC after error@500352813) -> (continuation of Deferred@2030199008 after retry RPC after error@1871462223), errback=(continuation of Deferred@1792783073 after retry RPC after error@1807620034) -> (continuation of Deferred@798263284 after retry RPC after error@302294493) -> (continuation of Deferred@629100998 after retry RPC after error@1046762156) -> (continuation of Deferred@902093235 after retry RPC after error@47823003) -> (continuation of Deferred@1576478823 after retry RPC after error@1092284487) -> (continuation of Deferred@1375772822 after retry RPC after error@500352813) -> (continuation of Deferred@2030199008 after retry RPC after error@1871462223)))
21:03:26.520 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1706758849(state=PENDING, result=null, callback=(continuation of Deferred@1792783073 after retry RPC after error@1807620034) -> (continuation of Deferred@798263284 after retry RPC after error@302294493) -> (continuation of Deferred@629100998 after retry RPC after error@1046762156) -> (continuation of Deferred@902093235 after retry RPC after error@47823003) -> (continuation of Deferred@1576478823 after retry RPC after error@1092284487) -> (continuation of Deferred@1375772822 after retry RPC after error@500352813) -> (continuation of Deferred@2030199008 after retry RPC after error@1871462223), errback=(continuation of Deferred@1792783073 after retry RPC after error@1807620034) -> (continuation of Deferred@798263284 after retry RPC after error@302294493) -> (continuation of Deferred@629100998 after retry RPC after error@1046762156) -> (continuation of Deferred@902093235 after retry RPC after error@47823003) -> (continuation of Deferred@1576478823 after retry RPC after error@1092284487) -> (continuation of Deferred@1375772822 after retry RPC after error@500352813) -> (continuation of Deferred@2030199008 after retry RPC after error@1871462223)))
21:03:26.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.740 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1097018263(state=PENDING, result=null, callback=(continuation of Deferred@1766528594 after retry RPC after error@1668951207) -> (continuation of Deferred@386811599 after retry RPC after error@1332344874) -> (continuation of Deferred@2124520979 after retry RPC after error@1284278900) -> (continuation of Deferred@693212436 after retry RPC after error@695832764) -> (continuation of Deferred@1541067398 after retry RPC after error@1940728995) -> (continuation of Deferred@1458700023 after retry RPC after error@862484709) -> (continuation of Deferred@1051448044 after retry RPC after error@620313006), errback=(continuation of Deferred@1766528594 after retry RPC after error@1668951207) -> (continuation of Deferred@386811599 after retry RPC after error@1332344874) -> (continuation of Deferred@2124520979 after retry RPC after error@1284278900) -> (continuation of Deferred@693212436 after retry RPC after error@695832764) -> (continuation of Deferred@1541067398 after retry RPC after error@1940728995) -> (continuation of Deferred@1458700023 after retry RPC after error@862484709) -> (continuation of Deferred@1051448044 after retry RPC after error@620313006)))
21:03:26.741 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1097018263(state=PENDING, result=null, callback=(continuation of Deferred@1766528594 after retry RPC after error@1668951207) -> (continuation of Deferred@386811599 after retry RPC after error@1332344874) -> (continuation of Deferred@2124520979 after retry RPC after error@1284278900) -> (continuation of Deferred@693212436 after retry RPC after error@695832764) -> (continuation of Deferred@1541067398 after retry RPC after error@1940728995) -> (continuation of Deferred@1458700023 after retry RPC after error@862484709) -> (continuation of Deferred@1051448044 after retry RPC after error@620313006), errback=(continuation of Deferred@1766528594 after retry RPC after error@1668951207) -> (continuation of Deferred@386811599 after retry RPC after error@1332344874) -> (continuation of Deferred@2124520979 after retry RPC after error@1284278900) -> (continuation of Deferred@693212436 after retry RPC after error@695832764) -> (continuation of Deferred@1541067398 after retry RPC after error@1940728995) -> (continuation of Deferred@1458700023 after retry RPC after error@862484709) -> (continuation of Deferred@1051448044 after retry RPC after error@620313006)))
21:03:26.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.779 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.780 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.780 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.781 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:26.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:26.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:26.939 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.939 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:26.939 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:26.940 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.017 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@928209074(state=PENDING, result=null, callback=(continuation of Deferred@715446618 after retry RPC after error@1510012020) -> (continuation of Deferred@506791171 after retry RPC after error@864966847) -> (continuation of Deferred@1072445853 after retry RPC after error@1653786944) -> (continuation of Deferred@1413076236 after retry RPC after error@355677332) -> (continuation of Deferred@1263388542 after retry RPC after error@1071767454) -> (continuation of Deferred@636157147 after retry RPC after error@703720294) -> (continuation of Deferred@211988241 after retry RPC after error@1032756276), errback=(continuation of Deferred@715446618 after retry RPC after error@1510012020) -> (continuation of Deferred@506791171 after retry RPC after error@864966847) -> (continuation of Deferred@1072445853 after retry RPC after error@1653786944) -> (continuation of Deferred@1413076236 after retry RPC after error@355677332) -> (continuation of Deferred@1263388542 after retry RPC after error@1071767454) -> (continuation of Deferred@636157147 after retry RPC after error@703720294) -> (continuation of Deferred@211988241 after retry RPC after error@1032756276)))
21:03:27.018 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:27.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.200 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@532700255(state=PENDING, result=null, callback=(continuation of Deferred@2005922891 after retry RPC after error@2102231631) -> (continuation of Deferred@1774368678 after retry RPC after error@1954360421) -> (continuation of Deferred@2086183693 after retry RPC after error@1830160691) -> (continuation of Deferred@295042224 after retry RPC after error@202831075) -> (continuation of Deferred@904887975 after retry RPC after error@1752016568) -> (continuation of Deferred@1445177209 after retry RPC after error@1831465138) -> (continuation of Deferred@603828726 after retry RPC after error@648055639), errback=(continuation of Deferred@2005922891 after retry RPC after error@2102231631) -> (continuation of Deferred@1774368678 after retry RPC after error@1954360421) -> (continuation of Deferred@2086183693 after retry RPC after error@1830160691) -> (continuation of Deferred@295042224 after retry RPC after error@202831075) -> (continuation of Deferred@904887975 after retry RPC after error@1752016568) -> (continuation of Deferred@1445177209 after retry RPC after error@1831465138) -> (continuation of Deferred@603828726 after retry RPC after error@648055639)))
21:03:27.200 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@532700255(state=PENDING, result=null, callback=(continuation of Deferred@2005922891 after retry RPC after error@2102231631) -> (continuation of Deferred@1774368678 after retry RPC after error@1954360421) -> (continuation of Deferred@2086183693 after retry RPC after error@1830160691) -> (continuation of Deferred@295042224 after retry RPC after error@202831075) -> (continuation of Deferred@904887975 after retry RPC after error@1752016568) -> (continuation of Deferred@1445177209 after retry RPC after error@1831465138) -> (continuation of Deferred@603828726 after retry RPC after error@648055639), errback=(continuation of Deferred@2005922891 after retry RPC after error@2102231631) -> (continuation of Deferred@1774368678 after retry RPC after error@1954360421) -> (continuation of Deferred@2086183693 after retry RPC after error@1830160691) -> (continuation of Deferred@295042224 after retry RPC after error@202831075) -> (continuation of Deferred@904887975 after retry RPC after error@1752016568) -> (continuation of Deferred@1445177209 after retry RPC after error@1831465138) -> (continuation of Deferred@603828726 after retry RPC after error@648055639)))
21:03:27.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.340 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.420 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@225158684(state=PENDING, result=null, callback=(continuation of Deferred@193205469 after retry RPC after error@1362473703) -> (continuation of Deferred@1667040536 after retry RPC after error@861279273) -> (continuation of Deferred@417320034 after retry RPC after error@1686238996) -> (continuation of Deferred@1870613337 after retry RPC after error@1812725187) -> (continuation of Deferred@181506300 after retry RPC after error@1636040677) -> (continuation of Deferred@1957909775 after retry RPC after error@1197824959), errback=(continuation of Deferred@193205469 after retry RPC after error@1362473703) -> (continuation of Deferred@1667040536 after retry RPC after error@861279273) -> (continuation of Deferred@417320034 after retry RPC after error@1686238996) -> (continuation of Deferred@1870613337 after retry RPC after error@1812725187) -> (continuation of Deferred@181506300 after retry RPC after error@1636040677) -> (continuation of Deferred@1957909775 after retry RPC after error@1197824959)))
21:03:27.420 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@225158684(state=PENDING, result=null, callback=(continuation of Deferred@193205469 after retry RPC after error@1362473703) -> (continuation of Deferred@1667040536 after retry RPC after error@861279273) -> (continuation of Deferred@417320034 after retry RPC after error@1686238996) -> (continuation of Deferred@1870613337 after retry RPC after error@1812725187) -> (continuation of Deferred@181506300 after retry RPC after error@1636040677) -> (continuation of Deferred@1957909775 after retry RPC after error@1197824959), errback=(continuation of Deferred@193205469 after retry RPC after error@1362473703) -> (continuation of Deferred@1667040536 after retry RPC after error@861279273) -> (continuation of Deferred@417320034 after retry RPC after error@1686238996) -> (continuation of Deferred@1870613337 after retry RPC after error@1812725187) -> (continuation of Deferred@181506300 after retry RPC after error@1636040677) -> (continuation of Deferred@1957909775 after retry RPC after error@1197824959)))
21:03:27.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:27.538534 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:27.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:27.540951 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:27.542248 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:03:27.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.560 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.560 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.579 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.659 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1352820268(state=PENDING, result=null, callback=(continuation of Deferred@1518851380 after retry RPC after error@212840528) -> (continuation of Deferred@1918596682 after retry RPC after error@822865195) -> (continuation of Deferred@1438752843 after retry RPC after error@605927315) -> (continuation of Deferred@224226428 after retry RPC after error@1674689097) -> (continuation of Deferred@994745728 after retry RPC after error@1923220565) -> (continuation of Deferred@343582961 after retry RPC after error@2048351173) -> (continuation of Deferred@358510490 after retry RPC after error@2000441800), errback=(continuation of Deferred@1518851380 after retry RPC after error@212840528) -> (continuation of Deferred@1918596682 after retry RPC after error@822865195) -> (continuation of Deferred@1438752843 after retry RPC after error@605927315) -> (continuation of Deferred@224226428 after retry RPC after error@1674689097) -> (continuation of Deferred@994745728 after retry RPC after error@1923220565) -> (continuation of Deferred@343582961 after retry RPC after error@2048351173) -> (continuation of Deferred@358510490 after retry RPC after error@2000441800)))
21:03:27.660 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1352820268(state=PENDING, result=null, callback=(continuation of Deferred@1518851380 after retry RPC after error@212840528) -> (continuation of Deferred@1918596682 after retry RPC after error@822865195) -> (continuation of Deferred@1438752843 after retry RPC after error@605927315) -> (continuation of Deferred@224226428 after retry RPC after error@1674689097) -> (continuation of Deferred@994745728 after retry RPC after error@1923220565) -> (continuation of Deferred@343582961 after retry RPC after error@2048351173) -> (continuation of Deferred@358510490 after retry RPC after error@2000441800), errback=(continuation of Deferred@1518851380 after retry RPC after error@212840528) -> (continuation of Deferred@1918596682 after retry RPC after error@822865195) -> (continuation of Deferred@1438752843 after retry RPC after error@605927315) -> (continuation of Deferred@224226428 after retry RPC after error@1674689097) -> (continuation of Deferred@994745728 after retry RPC after error@1923220565) -> (continuation of Deferred@343582961 after retry RPC after error@2048351173) -> (continuation of Deferred@358510490 after retry RPC after error@2000441800)))
21:03:27.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.698 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:27.736141 4117 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:27.736318 4117 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:27.736791 4117 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:27.737650 3320 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111) [suppressed 11 similar messages]
21:03:27.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:27.738965 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:27.739 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.739 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:27.739857 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:27.740020 3322 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:27.740233 4117 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:27.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.899 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@137232474(state=PENDING, result=null, callback=(continuation of Deferred@588866721 after retry RPC after error@1906808834) -> (continuation of Deferred@1672884559 after retry RPC after error@467849651) -> (continuation of Deferred@1941344770 after retry RPC after error@1042047998) -> (continuation of Deferred@1819414843 after retry RPC after error@1636842063) -> (continuation of Deferred@653361917 after retry RPC after error@2066679374) -> (continuation of Deferred@120077273 after retry RPC after error@1728719665) -> (continuation of Deferred@1304869703 after retry RPC after error@1322830279) -> (continuation of Deferred@149476662 after retry RPC after error@1041048761), errback=(continuation of Deferred@588866721 after retry RPC after error@1906808834) -> (continuation of Deferred@1672884559 after retry RPC after error@467849651) -> (continuation of Deferred@1941344770 after retry RPC after error@1042047998) -> (continuation of Deferred@1819414843 after retry RPC after error@1636842063) -> (continuation of Deferred@653361917 after retry RPC after error@2066679374) -> (continuation of Deferred@120077273 after retry RPC after error@1728719665) -> (continuation of Deferred@1304869703 after retry RPC after error@1322830279) -> (continuation of Deferred@149476662 after retry RPC after error@1041048761)))
21:03:27.900 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@137232474(state=PENDING, result=null, callback=(continuation of Deferred@588866721 after retry RPC after error@1906808834) -> (continuation of Deferred@1672884559 after retry RPC after error@467849651) -> (continuation of Deferred@1941344770 after retry RPC after error@1042047998) -> (continuation of Deferred@1819414843 after retry RPC after error@1636842063) -> (continuation of Deferred@653361917 after retry RPC after error@2066679374) -> (continuation of Deferred@120077273 after retry RPC after error@1728719665) -> (continuation of Deferred@1304869703 after retry RPC after error@1322830279) -> (continuation of Deferred@149476662 after retry RPC after error@1041048761), errback=(continuation of Deferred@588866721 after retry RPC after error@1906808834) -> (continuation of Deferred@1672884559 after retry RPC after error@467849651) -> (continuation of Deferred@1941344770 after retry RPC after error@1042047998) -> (continuation of Deferred@1819414843 after retry RPC after error@1636842063) -> (continuation of Deferred@653361917 after retry RPC after error@2066679374) -> (continuation of Deferred@120077273 after retry RPC after error@1728719665) -> (continuation of Deferred@1304869703 after retry RPC after error@1322830279) -> (continuation of Deferred@149476662 after retry RPC after error@1041048761)))
21:03:27.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.938 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.980 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:27.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:27.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:27.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:27.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:27.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.140 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1294722724(state=PENDING, result=null, callback=(continuation of Deferred@879356734 after retry RPC after error@1128122894) -> (continuation of Deferred@293971680 after retry RPC after error@59342002) -> (continuation of Deferred@2026002406 after retry RPC after error@1434619722) -> (continuation of Deferred@1762431420 after retry RPC after error@556276931) -> (continuation of Deferred@1262420583 after retry RPC after error@176565397) -> (continuation of Deferred@118152456 after retry RPC after error@1722095406) -> (continuation of Deferred@1429867943 after retry RPC after error@1860399377), errback=(continuation of Deferred@879356734 after retry RPC after error@1128122894) -> (continuation of Deferred@293971680 after retry RPC after error@59342002) -> (continuation of Deferred@2026002406 after retry RPC after error@1434619722) -> (continuation of Deferred@1762431420 after retry RPC after error@556276931) -> (continuation of Deferred@1262420583 after retry RPC after error@176565397) -> (continuation of Deferred@118152456 after retry RPC after error@1722095406) -> (continuation of Deferred@1429867943 after retry RPC after error@1860399377)))
21:03:28.141 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1294722724(state=PENDING, result=null, callback=(continuation of Deferred@879356734 after retry RPC after error@1128122894) -> (continuation of Deferred@293971680 after retry RPC after error@59342002) -> (continuation of Deferred@2026002406 after retry RPC after error@1434619722) -> (continuation of Deferred@1762431420 after retry RPC after error@556276931) -> (continuation of Deferred@1262420583 after retry RPC after error@176565397) -> (continuation of Deferred@118152456 after retry RPC after error@1722095406) -> (continuation of Deferred@1429867943 after retry RPC after error@1860399377), errback=(continuation of Deferred@879356734 after retry RPC after error@1128122894) -> (continuation of Deferred@293971680 after retry RPC after error@59342002) -> (continuation of Deferred@2026002406 after retry RPC after error@1434619722) -> (continuation of Deferred@1762431420 after retry RPC after error@556276931) -> (continuation of Deferred@1262420583 after retry RPC after error@176565397) -> (continuation of Deferred@118152456 after retry RPC after error@1722095406) -> (continuation of Deferred@1429867943 after retry RPC after error@1860399377)))
21:03:28.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.179 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.179 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.179 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.239 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.240 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.240 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.359 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2110432941(state=PENDING, result=null, callback=(continuation of Deferred@1702326430 after retry RPC after error@774441456) -> (continuation of Deferred@753149183 after retry RPC after error@78839808) -> (continuation of Deferred@1511945096 after retry RPC after error@1008637874) -> (continuation of Deferred@827363090 after retry RPC after error@763308600) -> (continuation of Deferred@72654380 after retry RPC after error@1279207955) -> (continuation of Deferred@2032276633 after retry RPC after error@769187695) -> (continuation of Deferred@224235782 after retry RPC after error@948395276), errback=(continuation of Deferred@1702326430 after retry RPC after error@774441456) -> (continuation of Deferred@753149183 after retry RPC after error@78839808) -> (continuation of Deferred@1511945096 after retry RPC after error@1008637874) -> (continuation of Deferred@827363090 after retry RPC after error@763308600) -> (continuation of Deferred@72654380 after retry RPC after error@1279207955) -> (continuation of Deferred@2032276633 after retry RPC after error@769187695) -> (continuation of Deferred@224235782 after retry RPC after error@948395276)))
21:03:28.360 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2110432941(state=PENDING, result=null, callback=(continuation of Deferred@1702326430 after retry RPC after error@774441456) -> (continuation of Deferred@753149183 after retry RPC after error@78839808) -> (continuation of Deferred@1511945096 after retry RPC after error@1008637874) -> (continuation of Deferred@827363090 after retry RPC after error@763308600) -> (continuation of Deferred@72654380 after retry RPC after error@1279207955) -> (continuation of Deferred@2032276633 after retry RPC after error@769187695) -> (continuation of Deferred@224235782 after retry RPC after error@948395276), errback=(continuation of Deferred@1702326430 after retry RPC after error@774441456) -> (continuation of Deferred@753149183 after retry RPC after error@78839808) -> (continuation of Deferred@1511945096 after retry RPC after error@1008637874) -> (continuation of Deferred@827363090 after retry RPC after error@763308600) -> (continuation of Deferred@72654380 after retry RPC after error@1279207955) -> (continuation of Deferred@2032276633 after retry RPC after error@769187695) -> (continuation of Deferred@224235782 after retry RPC after error@948395276)))
21:03:28.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.378 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.520 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2094555318(state=PENDING, result=null, callback=(continuation of Deferred@896655060 after retry RPC after error@567391046) -> (continuation of Deferred@5175704 after retry RPC after error@1068299511) -> (continuation of Deferred@76548776 after retry RPC after error@678941439) -> (continuation of Deferred@1602382928 after retry RPC after error@131145755) -> (continuation of Deferred@51425027 after retry RPC after error@359032244) -> (continuation of Deferred@1506442148 after retry RPC after error@1469401470), errback=(continuation of Deferred@896655060 after retry RPC after error@567391046) -> (continuation of Deferred@5175704 after retry RPC after error@1068299511) -> (continuation of Deferred@76548776 after retry RPC after error@678941439) -> (continuation of Deferred@1602382928 after retry RPC after error@131145755) -> (continuation of Deferred@51425027 after retry RPC after error@359032244) -> (continuation of Deferred@1506442148 after retry RPC after error@1469401470)))
21:03:28.521 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2094555318(state=PENDING, result=null, callback=(continuation of Deferred@896655060 after retry RPC after error@567391046) -> (continuation of Deferred@5175704 after retry RPC after error@1068299511) -> (continuation of Deferred@76548776 after retry RPC after error@678941439) -> (continuation of Deferred@1602382928 after retry RPC after error@131145755) -> (continuation of Deferred@51425027 after retry RPC after error@359032244) -> (continuation of Deferred@1506442148 after retry RPC after error@1469401470), errback=(continuation of Deferred@896655060 after retry RPC after error@567391046) -> (continuation of Deferred@5175704 after retry RPC after error@1068299511) -> (continuation of Deferred@76548776 after retry RPC after error@678941439) -> (continuation of Deferred@1602382928 after retry RPC after error@131145755) -> (continuation of Deferred@51425027 after retry RPC after error@359032244) -> (continuation of Deferred@1506442148 after retry RPC after error@1469401470)))
21:03:28.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:28.544031 4125 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:28.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:28.544224 4125 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:28.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:28.544839 4125 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:28.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:28.546782 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:28.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:28.547024 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:28.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:28.547153 3322 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:28.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:28.547430 4125 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:28.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.599 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.599 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.599 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.600 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.600 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.600 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.600 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.660 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.720 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1386543445(state=PENDING, result=null, callback=(continuation of Deferred@1828145072 after retry RPC after error@1334944336) -> (continuation of Deferred@1730317218 after retry RPC after error@305768514) -> (continuation of Deferred@1143322482 after retry RPC after error@107224708) -> (continuation of Deferred@1978198641 after retry RPC after error@718022454) -> (continuation of Deferred@1127979834 after retry RPC after error@1695366631) -> (continuation of Deferred@1468332117 after retry RPC after error@473595908) -> (continuation of Deferred@54999794 after retry RPC after error@254401912), errback=(continuation of Deferred@1828145072 after retry RPC after error@1334944336) -> (continuation of Deferred@1730317218 after retry RPC after error@305768514) -> (continuation of Deferred@1143322482 after retry RPC after error@107224708) -> (continuation of Deferred@1978198641 after retry RPC after error@718022454) -> (continuation of Deferred@1127979834 after retry RPC after error@1695366631) -> (continuation of Deferred@1468332117 after retry RPC after error@473595908) -> (continuation of Deferred@54999794 after retry RPC after error@254401912)))
21:03:28.720 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1386543445(state=PENDING, result=null, callback=(continuation of Deferred@1828145072 after retry RPC after error@1334944336) -> (continuation of Deferred@1730317218 after retry RPC after error@305768514) -> (continuation of Deferred@1143322482 after retry RPC after error@107224708) -> (continuation of Deferred@1978198641 after retry RPC after error@718022454) -> (continuation of Deferred@1127979834 after retry RPC after error@1695366631) -> (continuation of Deferred@1468332117 after retry RPC after error@473595908) -> (continuation of Deferred@54999794 after retry RPC after error@254401912), errback=(continuation of Deferred@1828145072 after retry RPC after error@1334944336) -> (continuation of Deferred@1730317218 after retry RPC after error@305768514) -> (continuation of Deferred@1143322482 after retry RPC after error@107224708) -> (continuation of Deferred@1978198641 after retry RPC after error@718022454) -> (continuation of Deferred@1127979834 after retry RPC after error@1695366631) -> (continuation of Deferred@1468332117 after retry RPC after error@473595908) -> (continuation of Deferred@54999794 after retry RPC after error@254401912)))
21:03:28.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.819 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.900 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@126500560(state=PENDING, result=null, callback=(continuation of Deferred@1891783437 after retry RPC after error@1526573090) -> (continuation of Deferred@2119647626 after retry RPC after error@687709230) -> (continuation of Deferred@1254488071 after retry RPC after error@2110715333) -> (continuation of Deferred@970039527 after retry RPC after error@2097760031) -> (continuation of Deferred@409570681 after retry RPC after error@310588268) -> (continuation of Deferred@643919683 after retry RPC after error@292025647) -> (continuation of Deferred@1160880191 after retry RPC after error@1319302054), errback=(continuation of Deferred@1891783437 after retry RPC after error@1526573090) -> (continuation of Deferred@2119647626 after retry RPC after error@687709230) -> (continuation of Deferred@1254488071 after retry RPC after error@2110715333) -> (continuation of Deferred@970039527 after retry RPC after error@2097760031) -> (continuation of Deferred@409570681 after retry RPC after error@310588268) -> (continuation of Deferred@643919683 after retry RPC after error@292025647) -> (continuation of Deferred@1160880191 after retry RPC after error@1319302054)))
21:03:28.900 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@126500560(state=PENDING, result=null, callback=(continuation of Deferred@1891783437 after retry RPC after error@1526573090) -> (continuation of Deferred@2119647626 after retry RPC after error@687709230) -> (continuation of Deferred@1254488071 after retry RPC after error@2110715333) -> (continuation of Deferred@970039527 after retry RPC after error@2097760031) -> (continuation of Deferred@409570681 after retry RPC after error@310588268) -> (continuation of Deferred@643919683 after retry RPC after error@292025647) -> (continuation of Deferred@1160880191 after retry RPC after error@1319302054), errback=(continuation of Deferred@1891783437 after retry RPC after error@1526573090) -> (continuation of Deferred@2119647626 after retry RPC after error@687709230) -> (continuation of Deferred@1254488071 after retry RPC after error@2110715333) -> (continuation of Deferred@970039527 after retry RPC after error@2097760031) -> (continuation of Deferred@409570681 after retry RPC after error@310588268) -> (continuation of Deferred@643919683 after retry RPC after error@292025647) -> (continuation of Deferred@1160880191 after retry RPC after error@1319302054)))
21:03:28.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.978 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:28.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:28.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:28.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:28.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:28.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.100 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1451846711(state=PENDING, result=null, callback=(continuation of Deferred@342293593 after retry RPC after error@2025891696) -> (continuation of Deferred@95565517 after retry RPC after error@415697560) -> (continuation of Deferred@996190701 after retry RPC after error@1181115094) -> (continuation of Deferred@1391865993 after retry RPC after error@1925542789) -> (continuation of Deferred@1280611313 after retry RPC after error@961620095) -> (continuation of Deferred@1514157476 after retry RPC after error@54537266) -> (continuation of Deferred@1810099376 after retry RPC after error@1539388259), errback=(continuation of Deferred@342293593 after retry RPC after error@2025891696) -> (continuation of Deferred@95565517 after retry RPC after error@415697560) -> (continuation of Deferred@996190701 after retry RPC after error@1181115094) -> (continuation of Deferred@1391865993 after retry RPC after error@1925542789) -> (continuation of Deferred@1280611313 after retry RPC after error@961620095) -> (continuation of Deferred@1514157476 after retry RPC after error@54537266) -> (continuation of Deferred@1810099376 after retry RPC after error@1539388259)))
21:03:29.101 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1451846711(state=PENDING, result=null, callback=(continuation of Deferred@342293593 after retry RPC after error@2025891696) -> (continuation of Deferred@95565517 after retry RPC after error@415697560) -> (continuation of Deferred@996190701 after retry RPC after error@1181115094) -> (continuation of Deferred@1391865993 after retry RPC after error@1925542789) -> (continuation of Deferred@1280611313 after retry RPC after error@961620095) -> (continuation of Deferred@1514157476 after retry RPC after error@54537266) -> (continuation of Deferred@1810099376 after retry RPC after error@1539388259), errback=(continuation of Deferred@342293593 after retry RPC after error@2025891696) -> (continuation of Deferred@95565517 after retry RPC after error@415697560) -> (continuation of Deferred@996190701 after retry RPC after error@1181115094) -> (continuation of Deferred@1391865993 after retry RPC after error@1925542789) -> (continuation of Deferred@1280611313 after retry RPC after error@961620095) -> (continuation of Deferred@1514157476 after retry RPC after error@54537266) -> (continuation of Deferred@1810099376 after retry RPC after error@1539388259)))
21:03:29.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.179 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.179 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.340 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@86522876(state=PENDING, result=null, callback=(continuation of Deferred@1430744971 after retry RPC after error@419728465) -> (continuation of Deferred@1436611321 after retry RPC after error@286508138) -> (continuation of Deferred@2143738612 after retry RPC after error@1109243737) -> (continuation of Deferred@284730073 after retry RPC after error@533136440) -> (continuation of Deferred@1453288526 after retry RPC after error@2031599291) -> (continuation of Deferred@741094698 after retry RPC after error@200082003) -> (continuation of Deferred@649982588 after retry RPC after error@1654354550) -> (continuation of Deferred@802393706 after retry RPC after error@1288695385), errback=(continuation of Deferred@1430744971 after retry RPC after error@419728465) -> (continuation of Deferred@1436611321 after retry RPC after error@286508138) -> (continuation of Deferred@2143738612 after retry RPC after error@1109243737) -> (continuation of Deferred@284730073 after retry RPC after error@533136440) -> (continuation of Deferred@1453288526 after retry RPC after error@2031599291) -> (continuation of Deferred@741094698 after retry RPC after error@200082003) -> (continuation of Deferred@649982588 after retry RPC after error@1654354550) -> (continuation of Deferred@802393706 after retry RPC after error@1288695385)))
21:03:29.341 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@86522876(state=PENDING, result=null, callback=(continuation of Deferred@1430744971 after retry RPC after error@419728465) -> (continuation of Deferred@1436611321 after retry RPC after error@286508138) -> (continuation of Deferred@2143738612 after retry RPC after error@1109243737) -> (continuation of Deferred@284730073 after retry RPC after error@533136440) -> (continuation of Deferred@1453288526 after retry RPC after error@2031599291) -> (continuation of Deferred@741094698 after retry RPC after error@200082003) -> (continuation of Deferred@649982588 after retry RPC after error@1654354550) -> (continuation of Deferred@802393706 after retry RPC after error@1288695385), errback=(continuation of Deferred@1430744971 after retry RPC after error@419728465) -> (continuation of Deferred@1436611321 after retry RPC after error@286508138) -> (continuation of Deferred@2143738612 after retry RPC after error@1109243737) -> (continuation of Deferred@284730073 after retry RPC after error@533136440) -> (continuation of Deferred@1453288526 after retry RPC after error@2031599291) -> (continuation of Deferred@741094698 after retry RPC after error@200082003) -> (continuation of Deferred@649982588 after retry RPC after error@1654354550) -> (continuation of Deferred@802393706 after retry RPC after error@1288695385)))
21:03:29.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.559 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1059176388(state=PENDING, result=null, callback=(continuation of Deferred@946519592 after retry RPC after error@484357724) -> (continuation of Deferred@1005481013 after retry RPC after error@159591187) -> (continuation of Deferred@1693802485 after retry RPC after error@233859425) -> (continuation of Deferred@1161241390 after retry RPC after error@1434207016) -> (continuation of Deferred@465085573 after retry RPC after error@361594354) -> (continuation of Deferred@1767838270 after retry RPC after error@1577941310) -> (continuation of Deferred@5202678 after retry RPC after error@2127822304), errback=(continuation of Deferred@946519592 after retry RPC after error@484357724) -> (continuation of Deferred@1005481013 after retry RPC after error@159591187) -> (continuation of Deferred@1693802485 after retry RPC after error@233859425) -> (continuation of Deferred@1161241390 after retry RPC after error@1434207016) -> (continuation of Deferred@465085573 after retry RPC after error@361594354) -> (continuation of Deferred@1767838270 after retry RPC after error@1577941310) -> (continuation of Deferred@5202678 after retry RPC after error@2127822304)))
21:03:29.560 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1059176388(state=PENDING, result=null, callback=(continuation of Deferred@946519592 after retry RPC after error@484357724) -> (continuation of Deferred@1005481013 after retry RPC after error@159591187) -> (continuation of Deferred@1693802485 after retry RPC after error@233859425) -> (continuation of Deferred@1161241390 after retry RPC after error@1434207016) -> (continuation of Deferred@465085573 after retry RPC after error@361594354) -> (continuation of Deferred@1767838270 after retry RPC after error@1577941310) -> (continuation of Deferred@5202678 after retry RPC after error@2127822304), errback=(continuation of Deferred@946519592 after retry RPC after error@484357724) -> (continuation of Deferred@1005481013 after retry RPC after error@159591187) -> (continuation of Deferred@1693802485 after retry RPC after error@233859425) -> (continuation of Deferred@1161241390 after retry RPC after error@1434207016) -> (continuation of Deferred@465085573 after retry RPC after error@361594354) -> (continuation of Deferred@1767838270 after retry RPC after error@1577941310) -> (continuation of Deferred@5202678 after retry RPC after error@2127822304)))
21:03:29.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.799 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1623188482(state=PENDING, result=null, callback=(continuation of Deferred@1815187911 after retry RPC after error@615708023) -> (continuation of Deferred@1238943957 after retry RPC after error@1457043992) -> (continuation of Deferred@906471085 after retry RPC after error@95286359) -> (continuation of Deferred@171711063 after retry RPC after error@430406002) -> (continuation of Deferred@312378917 after retry RPC after error@2021377901) -> (continuation of Deferred@318739607 after retry RPC after error@816907091) -> (continuation of Deferred@1459098830 after retry RPC after error@1889464787), errback=(continuation of Deferred@1815187911 after retry RPC after error@615708023) -> (continuation of Deferred@1238943957 after retry RPC after error@1457043992) -> (continuation of Deferred@906471085 after retry RPC after error@95286359) -> (continuation of Deferred@171711063 after retry RPC after error@430406002) -> (continuation of Deferred@312378917 after retry RPC after error@2021377901) -> (continuation of Deferred@318739607 after retry RPC after error@816907091) -> (continuation of Deferred@1459098830 after retry RPC after error@1889464787)))
21:03:29.800 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1623188482(state=PENDING, result=null, callback=(continuation of Deferred@1815187911 after retry RPC after error@615708023) -> (continuation of Deferred@1238943957 after retry RPC after error@1457043992) -> (continuation of Deferred@906471085 after retry RPC after error@95286359) -> (continuation of Deferred@171711063 after retry RPC after error@430406002) -> (continuation of Deferred@312378917 after retry RPC after error@2021377901) -> (continuation of Deferred@318739607 after retry RPC after error@816907091) -> (continuation of Deferred@1459098830 after retry RPC after error@1889464787), errback=(continuation of Deferred@1815187911 after retry RPC after error@615708023) -> (continuation of Deferred@1238943957 after retry RPC after error@1457043992) -> (continuation of Deferred@906471085 after retry RPC after error@95286359) -> (continuation of Deferred@171711063 after retry RPC after error@430406002) -> (continuation of Deferred@312378917 after retry RPC after error@2021377901) -> (continuation of Deferred@318739607 after retry RPC after error@816907091) -> (continuation of Deferred@1459098830 after retry RPC after error@1889464787)))
21:03:29.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.859 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:29.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:29.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:29.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:29.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:29.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.078 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1101703812(state=PENDING, result=null, callback=(continuation of Deferred@1432224429 after retry RPC after error@1396864831) -> (continuation of Deferred@1748225597 after retry RPC after error@2086254389) -> (continuation of Deferred@1496811897 after retry RPC after error@420275951) -> (continuation of Deferred@1874362837 after retry RPC after error@1583964936) -> (continuation of Deferred@997213693 after retry RPC after error@1161164770) -> (continuation of Deferred@1856532664 after retry RPC after error@2068594502) -> (continuation of Deferred@1110098559 after retry RPC after error@684369727), errback=(continuation of Deferred@1432224429 after retry RPC after error@1396864831) -> (continuation of Deferred@1748225597 after retry RPC after error@2086254389) -> (continuation of Deferred@1496811897 after retry RPC after error@420275951) -> (continuation of Deferred@1874362837 after retry RPC after error@1583964936) -> (continuation of Deferred@997213693 after retry RPC after error@1161164770) -> (continuation of Deferred@1856532664 after retry RPC after error@2068594502) -> (continuation of Deferred@1110098559 after retry RPC after error@684369727)))
21:03:30.078 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:30.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.179 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.179 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.179 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.280 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@627617040(state=PENDING, result=null, callback=(continuation of Deferred@168224020 after retry RPC after error@170325268) -> (continuation of Deferred@256429871 after retry RPC after error@1864954875) -> (continuation of Deferred@1379632463 after retry RPC after error@1871201353) -> (continuation of Deferred@644931360 after retry RPC after error@242173758) -> (continuation of Deferred@7387713 after retry RPC after error@762776586) -> (continuation of Deferred@224288603 after retry RPC after error@2046603129) -> (continuation of Deferred@2086383797 after retry RPC after error@1775891560), errback=(continuation of Deferred@168224020 after retry RPC after error@170325268) -> (continuation of Deferred@256429871 after retry RPC after error@1864954875) -> (continuation of Deferred@1379632463 after retry RPC after error@1871201353) -> (continuation of Deferred@644931360 after retry RPC after error@242173758) -> (continuation of Deferred@7387713 after retry RPC after error@762776586) -> (continuation of Deferred@224288603 after retry RPC after error@2046603129) -> (continuation of Deferred@2086383797 after retry RPC after error@1775891560)))
21:03:30.281 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@627617040(state=PENDING, result=null, callback=(continuation of Deferred@168224020 after retry RPC after error@170325268) -> (continuation of Deferred@256429871 after retry RPC after error@1864954875) -> (continuation of Deferred@1379632463 after retry RPC after error@1871201353) -> (continuation of Deferred@644931360 after retry RPC after error@242173758) -> (continuation of Deferred@7387713 after retry RPC after error@762776586) -> (continuation of Deferred@224288603 after retry RPC after error@2046603129) -> (continuation of Deferred@2086383797 after retry RPC after error@1775891560), errback=(continuation of Deferred@168224020 after retry RPC after error@170325268) -> (continuation of Deferred@256429871 after retry RPC after error@1864954875) -> (continuation of Deferred@1379632463 after retry RPC after error@1871201353) -> (continuation of Deferred@644931360 after retry RPC after error@242173758) -> (continuation of Deferred@7387713 after retry RPC after error@762776586) -> (continuation of Deferred@224288603 after retry RPC after error@2046603129) -> (continuation of Deferred@2086383797 after retry RPC after error@1775891560)))
21:03:30.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.439 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.439 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.441 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.500 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@600287545(state=PENDING, result=null, callback=(continuation of Deferred@1496814029 after retry RPC after error@2024159536) -> (continuation of Deferred@1148958512 after retry RPC after error@1525371111) -> (continuation of Deferred@1060576142 after retry RPC after error@1989266145) -> (continuation of Deferred@1875517748 after retry RPC after error@1873160258) -> (continuation of Deferred@1758111059 after retry RPC after error@276085865) -> (continuation of Deferred@780790994 after retry RPC after error@1882400121) -> (continuation of Deferred@1299113489 after retry RPC after error@2026671137), errback=(continuation of Deferred@1496814029 after retry RPC after error@2024159536) -> (continuation of Deferred@1148958512 after retry RPC after error@1525371111) -> (continuation of Deferred@1060576142 after retry RPC after error@1989266145) -> (continuation of Deferred@1875517748 after retry RPC after error@1873160258) -> (continuation of Deferred@1758111059 after retry RPC after error@276085865) -> (continuation of Deferred@780790994 after retry RPC after error@1882400121) -> (continuation of Deferred@1299113489 after retry RPC after error@2026671137)))
21:03:30.500 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@600287545(state=PENDING, result=null, callback=(continuation of Deferred@1496814029 after retry RPC after error@2024159536) -> (continuation of Deferred@1148958512 after retry RPC after error@1525371111) -> (continuation of Deferred@1060576142 after retry RPC after error@1989266145) -> (continuation of Deferred@1875517748 after retry RPC after error@1873160258) -> (continuation of Deferred@1758111059 after retry RPC after error@276085865) -> (continuation of Deferred@780790994 after retry RPC after error@1882400121) -> (continuation of Deferred@1299113489 after retry RPC after error@2026671137), errback=(continuation of Deferred@1496814029 after retry RPC after error@2024159536) -> (continuation of Deferred@1148958512 after retry RPC after error@1525371111) -> (continuation of Deferred@1060576142 after retry RPC after error@1989266145) -> (continuation of Deferred@1875517748 after retry RPC after error@1873160258) -> (continuation of Deferred@1758111059 after retry RPC after error@276085865) -> (continuation of Deferred@780790994 after retry RPC after error@1882400121) -> (continuation of Deferred@1299113489 after retry RPC after error@2026671137)))
21:03:30.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.700 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@682744433(state=PENDING, result=null, callback=(continuation of Deferred@1024785017 after retry RPC after error@1129454351) -> (continuation of Deferred@1894580918 after retry RPC after error@1445538952) -> (continuation of Deferred@1291113222 after retry RPC after error@523056116) -> (continuation of Deferred@122518037 after retry RPC after error@404704024) -> (continuation of Deferred@1922684926 after retry RPC after error@73267402) -> (continuation of Deferred@1747761030 after retry RPC after error@168388112), errback=(continuation of Deferred@1024785017 after retry RPC after error@1129454351) -> (continuation of Deferred@1894580918 after retry RPC after error@1445538952) -> (continuation of Deferred@1291113222 after retry RPC after error@523056116) -> (continuation of Deferred@122518037 after retry RPC after error@404704024) -> (continuation of Deferred@1922684926 after retry RPC after error@73267402) -> (continuation of Deferred@1747761030 after retry RPC after error@168388112)))
21:03:30.701 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@682744433(state=PENDING, result=null, callback=(continuation of Deferred@1024785017 after retry RPC after error@1129454351) -> (continuation of Deferred@1894580918 after retry RPC after error@1445538952) -> (continuation of Deferred@1291113222 after retry RPC after error@523056116) -> (continuation of Deferred@122518037 after retry RPC after error@404704024) -> (continuation of Deferred@1922684926 after retry RPC after error@73267402) -> (continuation of Deferred@1747761030 after retry RPC after error@168388112), errback=(continuation of Deferred@1024785017 after retry RPC after error@1129454351) -> (continuation of Deferred@1894580918 after retry RPC after error@1445538952) -> (continuation of Deferred@1291113222 after retry RPC after error@523056116) -> (continuation of Deferred@122518037 after retry RPC after error@404704024) -> (continuation of Deferred@1922684926 after retry RPC after error@73267402) -> (continuation of Deferred@1747761030 after retry RPC after error@168388112)))
21:03:30.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.920 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@958509107(state=PENDING, result=null, callback=(continuation of Deferred@1832785317 after retry RPC after error@1906954432) -> (continuation of Deferred@1112139380 after retry RPC after error@1994225190) -> (continuation of Deferred@1809040997 after retry RPC after error@1413690383) -> (continuation of Deferred@1504973493 after retry RPC after error@1153774563) -> (continuation of Deferred@835728880 after retry RPC after error@2097056905) -> (continuation of Deferred@86382617 after retry RPC after error@1745975669) -> (continuation of Deferred@1482599986 after retry RPC after error@1347218292), errback=(continuation of Deferred@1832785317 after retry RPC after error@1906954432) -> (continuation of Deferred@1112139380 after retry RPC after error@1994225190) -> (continuation of Deferred@1809040997 after retry RPC after error@1413690383) -> (continuation of Deferred@1504973493 after retry RPC after error@1153774563) -> (continuation of Deferred@835728880 after retry RPC after error@2097056905) -> (continuation of Deferred@86382617 after retry RPC after error@1745975669) -> (continuation of Deferred@1482599986 after retry RPC after error@1347218292)))
21:03:30.921 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@958509107(state=PENDING, result=null, callback=(continuation of Deferred@1832785317 after retry RPC after error@1906954432) -> (continuation of Deferred@1112139380 after retry RPC after error@1994225190) -> (continuation of Deferred@1809040997 after retry RPC after error@1413690383) -> (continuation of Deferred@1504973493 after retry RPC after error@1153774563) -> (continuation of Deferred@835728880 after retry RPC after error@2097056905) -> (continuation of Deferred@86382617 after retry RPC after error@1745975669) -> (continuation of Deferred@1482599986 after retry RPC after error@1347218292), errback=(continuation of Deferred@1832785317 after retry RPC after error@1906954432) -> (continuation of Deferred@1112139380 after retry RPC after error@1994225190) -> (continuation of Deferred@1809040997 after retry RPC after error@1413690383) -> (continuation of Deferred@1504973493 after retry RPC after error@1153774563) -> (continuation of Deferred@835728880 after retry RPC after error@2097056905) -> (continuation of Deferred@86382617 after retry RPC after error@1745975669) -> (continuation of Deferred@1482599986 after retry RPC after error@1347218292)))
21:03:30.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:30.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:30.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:30.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:30.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:30.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.019 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.019 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.020 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.020 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.020 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.026 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.060 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.120 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [89ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [89ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@388832806(state=PENDING, result=null, callback=(continuation of Deferred@98499652 after retry RPC after error@1299146121) -> (continuation of Deferred@685841863 after retry RPC after error@1472391891) -> (continuation of Deferred@629936307 after retry RPC after error@326724262) -> (continuation of Deferred@861291051 after retry RPC after error@1929552608) -> (continuation of Deferred@919485883 after retry RPC after error@1849314718) -> (continuation of Deferred@1878789480 after retry RPC after error@152025045), errback=(continuation of Deferred@98499652 after retry RPC after error@1299146121) -> (continuation of Deferred@685841863 after retry RPC after error@1472391891) -> (continuation of Deferred@629936307 after retry RPC after error@326724262) -> (continuation of Deferred@861291051 after retry RPC after error@1929552608) -> (continuation of Deferred@919485883 after retry RPC after error@1849314718) -> (continuation of Deferred@1878789480 after retry RPC after error@152025045)))
21:03:31.121 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [89ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [89ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@388832806(state=PENDING, result=null, callback=(continuation of Deferred@98499652 after retry RPC after error@1299146121) -> (continuation of Deferred@685841863 after retry RPC after error@1472391891) -> (continuation of Deferred@629936307 after retry RPC after error@326724262) -> (continuation of Deferred@861291051 after retry RPC after error@1929552608) -> (continuation of Deferred@919485883 after retry RPC after error@1849314718) -> (continuation of Deferred@1878789480 after retry RPC after error@152025045), errback=(continuation of Deferred@98499652 after retry RPC after error@1299146121) -> (continuation of Deferred@685841863 after retry RPC after error@1472391891) -> (continuation of Deferred@629936307 after retry RPC after error@326724262) -> (continuation of Deferred@861291051 after retry RPC after error@1929552608) -> (continuation of Deferred@919485883 after retry RPC after error@1849314718) -> (continuation of Deferred@1878789480 after retry RPC after error@152025045)))
21:03:31.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.239 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.239 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.379 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.379 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.380 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.380 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1364479948(state=PENDING, result=null, callback=(continuation of Deferred@1496303182 after retry RPC after error@1077004844) -> (continuation of Deferred@1844485119 after retry RPC after error@514429858) -> (continuation of Deferred@965950351 after retry RPC after error@2000549435) -> (continuation of Deferred@519808064 after retry RPC after error@1285361854) -> (continuation of Deferred@1819416033 after retry RPC after error@1961273978) -> (continuation of Deferred@939313410 after retry RPC after error@322975651) -> (continuation of Deferred@1877075255 after retry RPC after error@781531083) -> (continuation of Deferred@2146459586 after retry RPC after error@1839174320), errback=(continuation of Deferred@1496303182 after retry RPC after error@1077004844) -> (continuation of Deferred@1844485119 after retry RPC after error@514429858) -> (continuation of Deferred@965950351 after retry RPC after error@2000549435) -> (continuation of Deferred@519808064 after retry RPC after error@1285361854) -> (continuation of Deferred@1819416033 after retry RPC after error@1961273978) -> (continuation of Deferred@939313410 after retry RPC after error@322975651) -> (continuation of Deferred@1877075255 after retry RPC after error@781531083) -> (continuation of Deferred@2146459586 after retry RPC after error@1839174320)))
21:03:31.381 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1364479948(state=PENDING, result=null, callback=(continuation of Deferred@1496303182 after retry RPC after error@1077004844) -> (continuation of Deferred@1844485119 after retry RPC after error@514429858) -> (continuation of Deferred@965950351 after retry RPC after error@2000549435) -> (continuation of Deferred@519808064 after retry RPC after error@1285361854) -> (continuation of Deferred@1819416033 after retry RPC after error@1961273978) -> (continuation of Deferred@939313410 after retry RPC after error@322975651) -> (continuation of Deferred@1877075255 after retry RPC after error@781531083) -> (continuation of Deferred@2146459586 after retry RPC after error@1839174320), errback=(continuation of Deferred@1496303182 after retry RPC after error@1077004844) -> (continuation of Deferred@1844485119 after retry RPC after error@514429858) -> (continuation of Deferred@965950351 after retry RPC after error@2000549435) -> (continuation of Deferred@519808064 after retry RPC after error@1285361854) -> (continuation of Deferred@1819416033 after retry RPC after error@1961273978) -> (continuation of Deferred@939313410 after retry RPC after error@322975651) -> (continuation of Deferred@1877075255 after retry RPC after error@781531083) -> (continuation of Deferred@2146459586 after retry RPC after error@1839174320)))
21:03:31.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.419 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.519 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.560 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.657 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1450102194(state=PENDING, result=null, callback=(continuation of Deferred@1551172333 after retry RPC after error@1463325041) -> (continuation of Deferred@2532019 after retry RPC after error@537259096) -> (continuation of Deferred@1418981599 after retry RPC after error@1194545189) -> (continuation of Deferred@1924485450 after retry RPC after error@2103247976) -> (continuation of Deferred@914560532 after retry RPC after error@1810680473) -> (continuation of Deferred@864811439 after retry RPC after error@643876338) -> (continuation of Deferred@150819996 after retry RPC after error@88665880) -> (continuation of Deferred@1768040607 after retry RPC after error@1233993492), errback=(continuation of Deferred@1551172333 after retry RPC after error@1463325041) -> (continuation of Deferred@2532019 after retry RPC after error@537259096) -> (continuation of Deferred@1418981599 after retry RPC after error@1194545189) -> (continuation of Deferred@1924485450 after retry RPC after error@2103247976) -> (continuation of Deferred@914560532 after retry RPC after error@1810680473) -> (continuation of Deferred@864811439 after retry RPC after error@643876338) -> (continuation of Deferred@150819996 after retry RPC after error@88665880) -> (continuation of Deferred@1768040607 after retry RPC after error@1233993492)))
21:03:31.658 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:31.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.699 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.720 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.739 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.819 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.819 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.920 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [28ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1462532808(state=PENDING, result=null, callback=(continuation of Deferred@512068878 after retry RPC after error@664755463) -> (continuation of Deferred@330547474 after retry RPC after error@670892950) -> (continuation of Deferred@354551706 after retry RPC after error@1216874332) -> (continuation of Deferred@1170138126 after retry RPC after error@168865436) -> (continuation of Deferred@260798099 after retry RPC after error@120660697) -> (continuation of Deferred@899492005 after retry RPC after error@989201604) -> (continuation of Deferred@1855836903 after retry RPC after error@1614025223), errback=(continuation of Deferred@512068878 after retry RPC after error@664755463) -> (continuation of Deferred@330547474 after retry RPC after error@670892950) -> (continuation of Deferred@354551706 after retry RPC after error@1216874332) -> (continuation of Deferred@1170138126 after retry RPC after error@168865436) -> (continuation of Deferred@260798099 after retry RPC after error@120660697) -> (continuation of Deferred@899492005 after retry RPC after error@989201604) -> (continuation of Deferred@1855836903 after retry RPC after error@1614025223)))
21:03:31.920 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [28ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1462532808(state=PENDING, result=null, callback=(continuation of Deferred@512068878 after retry RPC after error@664755463) -> (continuation of Deferred@330547474 after retry RPC after error@670892950) -> (continuation of Deferred@354551706 after retry RPC after error@1216874332) -> (continuation of Deferred@1170138126 after retry RPC after error@168865436) -> (continuation of Deferred@260798099 after retry RPC after error@120660697) -> (continuation of Deferred@899492005 after retry RPC after error@989201604) -> (continuation of Deferred@1855836903 after retry RPC after error@1614025223), errback=(continuation of Deferred@512068878 after retry RPC after error@664755463) -> (continuation of Deferred@330547474 after retry RPC after error@670892950) -> (continuation of Deferred@354551706 after retry RPC after error@1216874332) -> (continuation of Deferred@1170138126 after retry RPC after error@168865436) -> (continuation of Deferred@260798099 after retry RPC after error@120660697) -> (continuation of Deferred@899492005 after retry RPC after error@989201604) -> (continuation of Deferred@1855836903 after retry RPC after error@1614025223)))
21:03:31.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:31.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:31.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:31.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:31.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:31.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.019 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.100 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.100 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1246224892(state=PENDING, result=null, callback=(continuation of Deferred@682178475 after retry RPC after error@1865749582) -> (continuation of Deferred@1441952237 after retry RPC after error@550449838) -> (continuation of Deferred@1102728563 after retry RPC after error@107656814) -> (continuation of Deferred@1087492695 after retry RPC after error@433246881) -> (continuation of Deferred@177259837 after retry RPC after error@1494932423) -> (continuation of Deferred@387447257 after retry RPC after error@408905280), errback=(continuation of Deferred@682178475 after retry RPC after error@1865749582) -> (continuation of Deferred@1441952237 after retry RPC after error@550449838) -> (continuation of Deferred@1102728563 after retry RPC after error@107656814) -> (continuation of Deferred@1087492695 after retry RPC after error@433246881) -> (continuation of Deferred@177259837 after retry RPC after error@1494932423) -> (continuation of Deferred@387447257 after retry RPC after error@408905280)))
21:03:32.100 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1246224892(state=PENDING, result=null, callback=(continuation of Deferred@682178475 after retry RPC after error@1865749582) -> (continuation of Deferred@1441952237 after retry RPC after error@550449838) -> (continuation of Deferred@1102728563 after retry RPC after error@107656814) -> (continuation of Deferred@1087492695 after retry RPC after error@433246881) -> (continuation of Deferred@177259837 after retry RPC after error@1494932423) -> (continuation of Deferred@387447257 after retry RPC after error@408905280), errback=(continuation of Deferred@682178475 after retry RPC after error@1865749582) -> (continuation of Deferred@1441952237 after retry RPC after error@550449838) -> (continuation of Deferred@1102728563 after retry RPC after error@107656814) -> (continuation of Deferred@1087492695 after retry RPC after error@433246881) -> (continuation of Deferred@177259837 after retry RPC after error@1494932423) -> (continuation of Deferred@387447257 after retry RPC after error@408905280)))
21:03:32.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.300 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@193032441(state=PENDING, result=null, callback=(continuation of Deferred@1538386206 after retry RPC after error@510348640) -> (continuation of Deferred@1385951818 after retry RPC after error@1386042968) -> (continuation of Deferred@112241676 after retry RPC after error@1123344887) -> (continuation of Deferred@1341145665 after retry RPC after error@2011689585) -> (continuation of Deferred@1958010957 after retry RPC after error@296377461) -> (continuation of Deferred@1552283211 after retry RPC after error@347726800), errback=(continuation of Deferred@1538386206 after retry RPC after error@510348640) -> (continuation of Deferred@1385951818 after retry RPC after error@1386042968) -> (continuation of Deferred@112241676 after retry RPC after error@1123344887) -> (continuation of Deferred@1341145665 after retry RPC after error@2011689585) -> (continuation of Deferred@1958010957 after retry RPC after error@296377461) -> (continuation of Deferred@1552283211 after retry RPC after error@347726800)))
21:03:32.301 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@193032441(state=PENDING, result=null, callback=(continuation of Deferred@1538386206 after retry RPC after error@510348640) -> (continuation of Deferred@1385951818 after retry RPC after error@1386042968) -> (continuation of Deferred@112241676 after retry RPC after error@1123344887) -> (continuation of Deferred@1341145665 after retry RPC after error@2011689585) -> (continuation of Deferred@1958010957 after retry RPC after error@296377461) -> (continuation of Deferred@1552283211 after retry RPC after error@347726800), errback=(continuation of Deferred@1538386206 after retry RPC after error@510348640) -> (continuation of Deferred@1385951818 after retry RPC after error@1386042968) -> (continuation of Deferred@112241676 after retry RPC after error@1123344887) -> (continuation of Deferred@1341145665 after retry RPC after error@2011689585) -> (continuation of Deferred@1958010957 after retry RPC after error@296377461) -> (continuation of Deferred@1552283211 after retry RPC after error@347726800)))
21:03:32.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.379 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.379 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.400 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.400 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.400 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.500 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@198758403(state=PENDING, result=null, callback=(continuation of Deferred@524362543 after retry RPC after error@4628838) -> (continuation of Deferred@1362677237 after retry RPC after error@481122073) -> (continuation of Deferred@182667256 after retry RPC after error@1058769491) -> (continuation of Deferred@685489952 after retry RPC after error@1580736701) -> (continuation of Deferred@1316253701 after retry RPC after error@769536650) -> (continuation of Deferred@1485417134 after retry RPC after error@1194022090), errback=(continuation of Deferred@524362543 after retry RPC after error@4628838) -> (continuation of Deferred@1362677237 after retry RPC after error@481122073) -> (continuation of Deferred@182667256 after retry RPC after error@1058769491) -> (continuation of Deferred@685489952 after retry RPC after error@1580736701) -> (continuation of Deferred@1316253701 after retry RPC after error@769536650) -> (continuation of Deferred@1485417134 after retry RPC after error@1194022090)))
21:03:32.501 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@198758403(state=PENDING, result=null, callback=(continuation of Deferred@524362543 after retry RPC after error@4628838) -> (continuation of Deferred@1362677237 after retry RPC after error@481122073) -> (continuation of Deferred@182667256 after retry RPC after error@1058769491) -> (continuation of Deferred@685489952 after retry RPC after error@1580736701) -> (continuation of Deferred@1316253701 after retry RPC after error@769536650) -> (continuation of Deferred@1485417134 after retry RPC after error@1194022090), errback=(continuation of Deferred@524362543 after retry RPC after error@4628838) -> (continuation of Deferred@1362677237 after retry RPC after error@481122073) -> (continuation of Deferred@182667256 after retry RPC after error@1058769491) -> (continuation of Deferred@685489952 after retry RPC after error@1580736701) -> (continuation of Deferred@1316253701 after retry RPC after error@769536650) -> (continuation of Deferred@1485417134 after retry RPC after error@1194022090)))
21:03:32.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.539 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:32.566783 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:32.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:32.571192 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:32.572808 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:03:32.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.579 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.580 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.620 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.638 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.760 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1691562250(state=PENDING, result=null, callback=(continuation of Deferred@268285066 after retry RPC after error@899648450) -> (continuation of Deferred@583136977 after retry RPC after error@1609029858) -> (continuation of Deferred@1029471856 after retry RPC after error@1960059778) -> (continuation of Deferred@1494099127 after retry RPC after error@953145276) -> (continuation of Deferred@1723760939 after retry RPC after error@866346573) -> (continuation of Deferred@455137042 after retry RPC after error@1561625085) -> (continuation of Deferred@1337062573 after retry RPC after error@1728606744), errback=(continuation of Deferred@268285066 after retry RPC after error@899648450) -> (continuation of Deferred@583136977 after retry RPC after error@1609029858) -> (continuation of Deferred@1029471856 after retry RPC after error@1960059778) -> (continuation of Deferred@1494099127 after retry RPC after error@953145276) -> (continuation of Deferred@1723760939 after retry RPC after error@866346573) -> (continuation of Deferred@455137042 after retry RPC after error@1561625085) -> (continuation of Deferred@1337062573 after retry RPC after error@1728606744)))
21:03:32.761 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1691562250(state=PENDING, result=null, callback=(continuation of Deferred@268285066 after retry RPC after error@899648450) -> (continuation of Deferred@583136977 after retry RPC after error@1609029858) -> (continuation of Deferred@1029471856 after retry RPC after error@1960059778) -> (continuation of Deferred@1494099127 after retry RPC after error@953145276) -> (continuation of Deferred@1723760939 after retry RPC after error@866346573) -> (continuation of Deferred@455137042 after retry RPC after error@1561625085) -> (continuation of Deferred@1337062573 after retry RPC after error@1728606744), errback=(continuation of Deferred@268285066 after retry RPC after error@899648450) -> (continuation of Deferred@583136977 after retry RPC after error@1609029858) -> (continuation of Deferred@1029471856 after retry RPC after error@1960059778) -> (continuation of Deferred@1494099127 after retry RPC after error@953145276) -> (continuation of Deferred@1723760939 after retry RPC after error@866346573) -> (continuation of Deferred@455137042 after retry RPC after error@1561625085) -> (continuation of Deferred@1337062573 after retry RPC after error@1728606744)))
21:03:32.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.799 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.819 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:32.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:32.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:32.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:32.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:32.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.020 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1407595366(state=PENDING, result=null, callback=(continuation of Deferred@1672109275 after retry RPC after error@864858817) -> (continuation of Deferred@2078368964 after retry RPC after error@436082141) -> (continuation of Deferred@671342269 after retry RPC after error@2127522758) -> (continuation of Deferred@242804639 after retry RPC after error@1740721008) -> (continuation of Deferred@1367447299 after retry RPC after error@1410684846) -> (continuation of Deferred@496502444 after retry RPC after error@1894728868) -> (continuation of Deferred@764148789 after retry RPC after error@1407022605), errback=(continuation of Deferred@1672109275 after retry RPC after error@864858817) -> (continuation of Deferred@2078368964 after retry RPC after error@436082141) -> (continuation of Deferred@671342269 after retry RPC after error@2127522758) -> (continuation of Deferred@242804639 after retry RPC after error@1740721008) -> (continuation of Deferred@1367447299 after retry RPC after error@1410684846) -> (continuation of Deferred@496502444 after retry RPC after error@1894728868) -> (continuation of Deferred@764148789 after retry RPC after error@1407022605)))
21:03:33.020 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1407595366(state=PENDING, result=null, callback=(continuation of Deferred@1672109275 after retry RPC after error@864858817) -> (continuation of Deferred@2078368964 after retry RPC after error@436082141) -> (continuation of Deferred@671342269 after retry RPC after error@2127522758) -> (continuation of Deferred@242804639 after retry RPC after error@1740721008) -> (continuation of Deferred@1367447299 after retry RPC after error@1410684846) -> (continuation of Deferred@496502444 after retry RPC after error@1894728868) -> (continuation of Deferred@764148789 after retry RPC after error@1407022605), errback=(continuation of Deferred@1672109275 after retry RPC after error@864858817) -> (continuation of Deferred@2078368964 after retry RPC after error@436082141) -> (continuation of Deferred@671342269 after retry RPC after error@2127522758) -> (continuation of Deferred@242804639 after retry RPC after error@1740721008) -> (continuation of Deferred@1367447299 after retry RPC after error@1410684846) -> (continuation of Deferred@496502444 after retry RPC after error@1894728868) -> (continuation of Deferred@764148789 after retry RPC after error@1407022605)))
21:03:33.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.100 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.279 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.279 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.280 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@801409315(state=PENDING, result=null, callback=(continuation of Deferred@2045126311 after retry RPC after error@1263883084) -> (continuation of Deferred@1624812781 after retry RPC after error@1809759300) -> (continuation of Deferred@591752326 after retry RPC after error@1122731153) -> (continuation of Deferred@1743906789 after retry RPC after error@2043890627) -> (continuation of Deferred@1903835427 after retry RPC after error@1865147537) -> (continuation of Deferred@118351966 after retry RPC after error@1684219439) -> (continuation of Deferred@1156626593 after retry RPC after error@1959777218), errback=(continuation of Deferred@2045126311 after retry RPC after error@1263883084) -> (continuation of Deferred@1624812781 after retry RPC after error@1809759300) -> (continuation of Deferred@591752326 after retry RPC after error@1122731153) -> (continuation of Deferred@1743906789 after retry RPC after error@2043890627) -> (continuation of Deferred@1903835427 after retry RPC after error@1865147537) -> (continuation of Deferred@118351966 after retry RPC after error@1684219439) -> (continuation of Deferred@1156626593 after retry RPC after error@1959777218)))
21:03:33.281 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@801409315(state=PENDING, result=null, callback=(continuation of Deferred@2045126311 after retry RPC after error@1263883084) -> (continuation of Deferred@1624812781 after retry RPC after error@1809759300) -> (continuation of Deferred@591752326 after retry RPC after error@1122731153) -> (continuation of Deferred@1743906789 after retry RPC after error@2043890627) -> (continuation of Deferred@1903835427 after retry RPC after error@1865147537) -> (continuation of Deferred@118351966 after retry RPC after error@1684219439) -> (continuation of Deferred@1156626593 after retry RPC after error@1959777218), errback=(continuation of Deferred@2045126311 after retry RPC after error@1263883084) -> (continuation of Deferred@1624812781 after retry RPC after error@1809759300) -> (continuation of Deferred@591752326 after retry RPC after error@1122731153) -> (continuation of Deferred@1743906789 after retry RPC after error@2043890627) -> (continuation of Deferred@1903835427 after retry RPC after error@1865147537) -> (continuation of Deferred@118351966 after retry RPC after error@1684219439) -> (continuation of Deferred@1156626593 after retry RPC after error@1959777218)))
21:03:33.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.520 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1644480875(state=PENDING, result=null, callback=(continuation of Deferred@2039080641 after retry RPC after error@1876519343) -> (continuation of Deferred@1625729900 after retry RPC after error@619538826) -> (continuation of Deferred@308554808 after retry RPC after error@911522762) -> (continuation of Deferred@1664928679 after retry RPC after error@743591795) -> (continuation of Deferred@535077677 after retry RPC after error@217960944) -> (continuation of Deferred@161038907 after retry RPC after error@923712940) -> (continuation of Deferred@222928299 after retry RPC after error@1876541347), errback=(continuation of Deferred@2039080641 after retry RPC after error@1876519343) -> (continuation of Deferred@1625729900 after retry RPC after error@619538826) -> (continuation of Deferred@308554808 after retry RPC after error@911522762) -> (continuation of Deferred@1664928679 after retry RPC after error@743591795) -> (continuation of Deferred@535077677 after retry RPC after error@217960944) -> (continuation of Deferred@161038907 after retry RPC after error@923712940) -> (continuation of Deferred@222928299 after retry RPC after error@1876541347)))
21:03:33.520 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1644480875(state=PENDING, result=null, callback=(continuation of Deferred@2039080641 after retry RPC after error@1876519343) -> (continuation of Deferred@1625729900 after retry RPC after error@619538826) -> (continuation of Deferred@308554808 after retry RPC after error@911522762) -> (continuation of Deferred@1664928679 after retry RPC after error@743591795) -> (continuation of Deferred@535077677 after retry RPC after error@217960944) -> (continuation of Deferred@161038907 after retry RPC after error@923712940) -> (continuation of Deferred@222928299 after retry RPC after error@1876541347), errback=(continuation of Deferred@2039080641 after retry RPC after error@1876519343) -> (continuation of Deferred@1625729900 after retry RPC after error@619538826) -> (continuation of Deferred@308554808 after retry RPC after error@911522762) -> (continuation of Deferred@1664928679 after retry RPC after error@743591795) -> (continuation of Deferred@535077677 after retry RPC after error@217960944) -> (continuation of Deferred@161038907 after retry RPC after error@923712940) -> (continuation of Deferred@222928299 after retry RPC after error@1876541347)))
21:03:33.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.558 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.679 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.679 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.759 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@753925434(state=PENDING, result=null, callback=(continuation of Deferred@762015706 after retry RPC after error@1211992264) -> (continuation of Deferred@870456398 after retry RPC after error@1622614309) -> (continuation of Deferred@383031126 after retry RPC after error@847984268) -> (continuation of Deferred@287853823 after retry RPC after error@1221589494) -> (continuation of Deferred@2056501662 after retry RPC after error@289922838) -> (continuation of Deferred@1113273973 after retry RPC after error@1970038657) -> (continuation of Deferred@315912187 after retry RPC after error@1036303832), errback=(continuation of Deferred@762015706 after retry RPC after error@1211992264) -> (continuation of Deferred@870456398 after retry RPC after error@1622614309) -> (continuation of Deferred@383031126 after retry RPC after error@847984268) -> (continuation of Deferred@287853823 after retry RPC after error@1221589494) -> (continuation of Deferred@2056501662 after retry RPC after error@289922838) -> (continuation of Deferred@1113273973 after retry RPC after error@1970038657) -> (continuation of Deferred@315912187 after retry RPC after error@1036303832)))
21:03:33.760 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@753925434(state=PENDING, result=null, callback=(continuation of Deferred@762015706 after retry RPC after error@1211992264) -> (continuation of Deferred@870456398 after retry RPC after error@1622614309) -> (continuation of Deferred@383031126 after retry RPC after error@847984268) -> (continuation of Deferred@287853823 after retry RPC after error@1221589494) -> (continuation of Deferred@2056501662 after retry RPC after error@289922838) -> (continuation of Deferred@1113273973 after retry RPC after error@1970038657) -> (continuation of Deferred@315912187 after retry RPC after error@1036303832), errback=(continuation of Deferred@762015706 after retry RPC after error@1211992264) -> (continuation of Deferred@870456398 after retry RPC after error@1622614309) -> (continuation of Deferred@383031126 after retry RPC after error@847984268) -> (continuation of Deferred@287853823 after retry RPC after error@1221589494) -> (continuation of Deferred@2056501662 after retry RPC after error@289922838) -> (continuation of Deferred@1113273973 after retry RPC after error@1970038657) -> (continuation of Deferred@315912187 after retry RPC after error@1036303832)))
21:03:33.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.859 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.877 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.878 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.919 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.958 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:33.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:33.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:33.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:33.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:33.980 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.020 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=10, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1561417821(state=PENDING, result=null, callback=(continuation of Deferred@1824675223 after retry RPC after error@1645377618) -> (continuation of Deferred@178186258 after retry RPC after error@1562024620) -> (continuation of Deferred@596247157 after retry RPC after error@1378565822) -> (continuation of Deferred@797479569 after retry RPC after error@642687542) -> (continuation of Deferred@1436849440 after retry RPC after error@1584500708) -> (continuation of Deferred@889619105 after retry RPC after error@1711982885) -> (continuation of Deferred@393633105 after retry RPC after error@1168834292) -> (continuation of Deferred@1152941930 after retry RPC after error@1959511353) -> (continuation of Deferred@1374851620 after retry RPC after error@1128111024), errback=(continuation of Deferred@1824675223 after retry RPC after error@1645377618) -> (continuation of Deferred@178186258 after retry RPC after error@1562024620) -> (continuation of Deferred@596247157 after retry RPC after error@1378565822) -> (continuation of Deferred@797479569 after retry RPC after error@642687542) -> (continuation of Deferred@1436849440 after retry RPC after error@1584500708) -> (continuation of Deferred@889619105 after retry RPC after error@1711982885) -> (continuation of Deferred@393633105 after retry RPC after error@1168834292) -> (continuation of Deferred@1152941930 after retry RPC after error@1959511353) -> (continuation of Deferred@1374851620 after retry RPC after error@1128111024)))
21:03:34.020 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=10, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1561417821(state=PENDING, result=null, callback=(continuation of Deferred@1824675223 after retry RPC after error@1645377618) -> (continuation of Deferred@178186258 after retry RPC after error@1562024620) -> (continuation of Deferred@596247157 after retry RPC after error@1378565822) -> (continuation of Deferred@797479569 after retry RPC after error@642687542) -> (continuation of Deferred@1436849440 after retry RPC after error@1584500708) -> (continuation of Deferred@889619105 after retry RPC after error@1711982885) -> (continuation of Deferred@393633105 after retry RPC after error@1168834292) -> (continuation of Deferred@1152941930 after retry RPC after error@1959511353) -> (continuation of Deferred@1374851620 after retry RPC after error@1128111024), errback=(continuation of Deferred@1824675223 after retry RPC after error@1645377618) -> (continuation of Deferred@178186258 after retry RPC after error@1562024620) -> (continuation of Deferred@596247157 after retry RPC after error@1378565822) -> (continuation of Deferred@797479569 after retry RPC after error@642687542) -> (continuation of Deferred@1436849440 after retry RPC after error@1584500708) -> (continuation of Deferred@889619105 after retry RPC after error@1711982885) -> (continuation of Deferred@393633105 after retry RPC after error@1168834292) -> (continuation of Deferred@1152941930 after retry RPC after error@1959511353) -> (continuation of Deferred@1374851620 after retry RPC after error@1128111024)))
21:03:34.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.099 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.100 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.199 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.297 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1552678400(state=PENDING, result=null, callback=(continuation of Deferred@505033114 after retry RPC after error@2141061828) -> (continuation of Deferred@797562823 after retry RPC after error@745621448) -> (continuation of Deferred@1663579453 after retry RPC after error@1823441854) -> (continuation of Deferred@226051587 after retry RPC after error@730861411) -> (continuation of Deferred@162744049 after retry RPC after error@428967434) -> (continuation of Deferred@1579123301 after retry RPC after error@1977578962) -> (continuation of Deferred@1724395862 after retry RPC after error@1937529427), errback=(continuation of Deferred@505033114 after retry RPC after error@2141061828) -> (continuation of Deferred@797562823 after retry RPC after error@745621448) -> (continuation of Deferred@1663579453 after retry RPC after error@1823441854) -> (continuation of Deferred@226051587 after retry RPC after error@730861411) -> (continuation of Deferred@162744049 after retry RPC after error@428967434) -> (continuation of Deferred@1579123301 after retry RPC after error@1977578962) -> (continuation of Deferred@1724395862 after retry RPC after error@1937529427)))
21:03:34.298 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:34.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.499 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1301442512(state=PENDING, result=null, callback=(continuation of Deferred@1607972261 after retry RPC after error@2007989418) -> (continuation of Deferred@1551654342 after retry RPC after error@38688118) -> (continuation of Deferred@1624678919 after retry RPC after error@1813039259) -> (continuation of Deferred@1379674802 after retry RPC after error@573120614) -> (continuation of Deferred@2000316606 after retry RPC after error@486810322) -> (continuation of Deferred@786363002 after retry RPC after error@1159716311) -> (continuation of Deferred@1640247890 after retry RPC after error@1575652874), errback=(continuation of Deferred@1607972261 after retry RPC after error@2007989418) -> (continuation of Deferred@1551654342 after retry RPC after error@38688118) -> (continuation of Deferred@1624678919 after retry RPC after error@1813039259) -> (continuation of Deferred@1379674802 after retry RPC after error@573120614) -> (continuation of Deferred@2000316606 after retry RPC after error@486810322) -> (continuation of Deferred@786363002 after retry RPC after error@1159716311) -> (continuation of Deferred@1640247890 after retry RPC after error@1575652874)))
21:03:34.500 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1301442512(state=PENDING, result=null, callback=(continuation of Deferred@1607972261 after retry RPC after error@2007989418) -> (continuation of Deferred@1551654342 after retry RPC after error@38688118) -> (continuation of Deferred@1624678919 after retry RPC after error@1813039259) -> (continuation of Deferred@1379674802 after retry RPC after error@573120614) -> (continuation of Deferred@2000316606 after retry RPC after error@486810322) -> (continuation of Deferred@786363002 after retry RPC after error@1159716311) -> (continuation of Deferred@1640247890 after retry RPC after error@1575652874), errback=(continuation of Deferred@1607972261 after retry RPC after error@2007989418) -> (continuation of Deferred@1551654342 after retry RPC after error@38688118) -> (continuation of Deferred@1624678919 after retry RPC after error@1813039259) -> (continuation of Deferred@1379674802 after retry RPC after error@573120614) -> (continuation of Deferred@2000316606 after retry RPC after error@486810322) -> (continuation of Deferred@786363002 after retry RPC after error@1159716311) -> (continuation of Deferred@1640247890 after retry RPC after error@1575652874)))
21:03:34.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.519 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.539 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.777 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@918010353(state=PENDING, result=null, callback=(continuation of Deferred@546631485 after retry RPC after error@1469316906) -> (continuation of Deferred@22188820 after retry RPC after error@169990910) -> (continuation of Deferred@231420288 after retry RPC after error@1164572092) -> (continuation of Deferred@1345721398 after retry RPC after error@1422181092) -> (continuation of Deferred@14139085 after retry RPC after error@177827710) -> (continuation of Deferred@1794002367 after retry RPC after error@407859684) -> (continuation of Deferred@634619382 after retry RPC after error@1685122221), errback=(continuation of Deferred@546631485 after retry RPC after error@1469316906) -> (continuation of Deferred@22188820 after retry RPC after error@169990910) -> (continuation of Deferred@231420288 after retry RPC after error@1164572092) -> (continuation of Deferred@1345721398 after retry RPC after error@1422181092) -> (continuation of Deferred@14139085 after retry RPC after error@177827710) -> (continuation of Deferred@1794002367 after retry RPC after error@407859684) -> (continuation of Deferred@634619382 after retry RPC after error@1685122221)))
21:03:34.778 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:34.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.857 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.858 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:34.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:34.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:34.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:34.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:34.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.058 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1391562270(state=PENDING, result=null, callback=(continuation of Deferred@450768011 after retry RPC after error@27623115) -> (continuation of Deferred@1959885068 after retry RPC after error@713988342) -> (continuation of Deferred@1096766296 after retry RPC after error@1797664414) -> (continuation of Deferred@339747823 after retry RPC after error@1247232185) -> (continuation of Deferred@1898520376 after retry RPC after error@723619165) -> (continuation of Deferred@1047239821 after retry RPC after error@1321816433) -> (continuation of Deferred@2006996815 after retry RPC after error@1347184515), errback=(continuation of Deferred@450768011 after retry RPC after error@27623115) -> (continuation of Deferred@1959885068 after retry RPC after error@713988342) -> (continuation of Deferred@1096766296 after retry RPC after error@1797664414) -> (continuation of Deferred@339747823 after retry RPC after error@1247232185) -> (continuation of Deferred@1898520376 after retry RPC after error@723619165) -> (continuation of Deferred@1047239821 after retry RPC after error@1321816433) -> (continuation of Deferred@2006996815 after retry RPC after error@1347184515)))
21:03:35.059 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:35.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.259 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.259 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.259 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.260 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.279 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.279 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.279 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.319 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@688606118(state=PENDING, result=null, callback=(continuation of Deferred@1156817148 after retry RPC after error@1069707395) -> (continuation of Deferred@209627354 after retry RPC after error@2145096840) -> (continuation of Deferred@653668303 after retry RPC after error@169612794) -> (continuation of Deferred@1935496219 after retry RPC after error@1350693164) -> (continuation of Deferred@1193161892 after retry RPC after error@759650936) -> (continuation of Deferred@874720839 after retry RPC after error@1811194238) -> (continuation of Deferred@1568211888 after retry RPC after error@1257899773) -> (continuation of Deferred@1728465139 after retry RPC after error@691534396), errback=(continuation of Deferred@1156817148 after retry RPC after error@1069707395) -> (continuation of Deferred@209627354 after retry RPC after error@2145096840) -> (continuation of Deferred@653668303 after retry RPC after error@169612794) -> (continuation of Deferred@1935496219 after retry RPC after error@1350693164) -> (continuation of Deferred@1193161892 after retry RPC after error@759650936) -> (continuation of Deferred@874720839 after retry RPC after error@1811194238) -> (continuation of Deferred@1568211888 after retry RPC after error@1257899773) -> (continuation of Deferred@1728465139 after retry RPC after error@691534396)))
21:03:35.320 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@688606118(state=PENDING, result=null, callback=(continuation of Deferred@1156817148 after retry RPC after error@1069707395) -> (continuation of Deferred@209627354 after retry RPC after error@2145096840) -> (continuation of Deferred@653668303 after retry RPC after error@169612794) -> (continuation of Deferred@1935496219 after retry RPC after error@1350693164) -> (continuation of Deferred@1193161892 after retry RPC after error@759650936) -> (continuation of Deferred@874720839 after retry RPC after error@1811194238) -> (continuation of Deferred@1568211888 after retry RPC after error@1257899773) -> (continuation of Deferred@1728465139 after retry RPC after error@691534396), errback=(continuation of Deferred@1156817148 after retry RPC after error@1069707395) -> (continuation of Deferred@209627354 after retry RPC after error@2145096840) -> (continuation of Deferred@653668303 after retry RPC after error@169612794) -> (continuation of Deferred@1935496219 after retry RPC after error@1350693164) -> (continuation of Deferred@1193161892 after retry RPC after error@759650936) -> (continuation of Deferred@874720839 after retry RPC after error@1811194238) -> (continuation of Deferred@1568211888 after retry RPC after error@1257899773) -> (continuation of Deferred@1728465139 after retry RPC after error@691534396)))
21:03:35.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.339 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.357 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.358 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.579 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1398823251(state=PENDING, result=null, callback=(continuation of Deferred@455980584 after retry RPC after error@1075592789) -> (continuation of Deferred@772982132 after retry RPC after error@2137635620) -> (continuation of Deferred@151248407 after retry RPC after error@684505076) -> (continuation of Deferred@281935612 after retry RPC after error@575653722) -> (continuation of Deferred@892028514 after retry RPC after error@2023040036) -> (continuation of Deferred@122691540 after retry RPC after error@1543077235) -> (continuation of Deferred@881731640 after retry RPC after error@1701547657) -> (continuation of Deferred@1648526159 after retry RPC after error@1916827793), errback=(continuation of Deferred@455980584 after retry RPC after error@1075592789) -> (continuation of Deferred@772982132 after retry RPC after error@2137635620) -> (continuation of Deferred@151248407 after retry RPC after error@684505076) -> (continuation of Deferred@281935612 after retry RPC after error@575653722) -> (continuation of Deferred@892028514 after retry RPC after error@2023040036) -> (continuation of Deferred@122691540 after retry RPC after error@1543077235) -> (continuation of Deferred@881731640 after retry RPC after error@1701547657) -> (continuation of Deferred@1648526159 after retry RPC after error@1916827793)))
21:03:35.580 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1398823251(state=PENDING, result=null, callback=(continuation of Deferred@455980584 after retry RPC after error@1075592789) -> (continuation of Deferred@772982132 after retry RPC after error@2137635620) -> (continuation of Deferred@151248407 after retry RPC after error@684505076) -> (continuation of Deferred@281935612 after retry RPC after error@575653722) -> (continuation of Deferred@892028514 after retry RPC after error@2023040036) -> (continuation of Deferred@122691540 after retry RPC after error@1543077235) -> (continuation of Deferred@881731640 after retry RPC after error@1701547657) -> (continuation of Deferred@1648526159 after retry RPC after error@1916827793), errback=(continuation of Deferred@455980584 after retry RPC after error@1075592789) -> (continuation of Deferred@772982132 after retry RPC after error@2137635620) -> (continuation of Deferred@151248407 after retry RPC after error@684505076) -> (continuation of Deferred@281935612 after retry RPC after error@575653722) -> (continuation of Deferred@892028514 after retry RPC after error@2023040036) -> (continuation of Deferred@122691540 after retry RPC after error@1543077235) -> (continuation of Deferred@881731640 after retry RPC after error@1701547657) -> (continuation of Deferred@1648526159 after retry RPC after error@1916827793)))
21:03:35.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.717 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.760 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@732578382(state=PENDING, result=null, callback=(continuation of Deferred@1178640589 after retry RPC after error@401453902) -> (continuation of Deferred@415070924 after retry RPC after error@1635574695) -> (continuation of Deferred@586772877 after retry RPC after error@1492435295) -> (continuation of Deferred@736934807 after retry RPC after error@764085398) -> (continuation of Deferred@1142386099 after retry RPC after error@974631511) -> (continuation of Deferred@1953072769 after retry RPC after error@28545838), errback=(continuation of Deferred@1178640589 after retry RPC after error@401453902) -> (continuation of Deferred@415070924 after retry RPC after error@1635574695) -> (continuation of Deferred@586772877 after retry RPC after error@1492435295) -> (continuation of Deferred@736934807 after retry RPC after error@764085398) -> (continuation of Deferred@1142386099 after retry RPC after error@974631511) -> (continuation of Deferred@1953072769 after retry RPC after error@28545838)))
21:03:35.760 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@732578382(state=PENDING, result=null, callback=(continuation of Deferred@1178640589 after retry RPC after error@401453902) -> (continuation of Deferred@415070924 after retry RPC after error@1635574695) -> (continuation of Deferred@586772877 after retry RPC after error@1492435295) -> (continuation of Deferred@736934807 after retry RPC after error@764085398) -> (continuation of Deferred@1142386099 after retry RPC after error@974631511) -> (continuation of Deferred@1953072769 after retry RPC after error@28545838), errback=(continuation of Deferred@1178640589 after retry RPC after error@401453902) -> (continuation of Deferred@415070924 after retry RPC after error@1635574695) -> (continuation of Deferred@586772877 after retry RPC after error@1492435295) -> (continuation of Deferred@736934807 after retry RPC after error@764085398) -> (continuation of Deferred@1142386099 after retry RPC after error@974631511) -> (continuation of Deferred@1953072769 after retry RPC after error@28545838)))
21:03:35.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.859 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.960 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1484244187(state=PENDING, result=null, callback=(continuation of Deferred@30749985 after retry RPC after error@1126640954) -> (continuation of Deferred@1814193286 after retry RPC after error@177546437) -> (continuation of Deferred@574715019 after retry RPC after error@1492501252) -> (continuation of Deferred@663521992 after retry RPC after error@68952560) -> (continuation of Deferred@768669908 after retry RPC after error@1531700105) -> (continuation of Deferred@408086754 after retry RPC after error@2099534265) -> (continuation of Deferred@1461196945 after retry RPC after error@1405873744), errback=(continuation of Deferred@30749985 after retry RPC after error@1126640954) -> (continuation of Deferred@1814193286 after retry RPC after error@177546437) -> (continuation of Deferred@574715019 after retry RPC after error@1492501252) -> (continuation of Deferred@663521992 after retry RPC after error@68952560) -> (continuation of Deferred@768669908 after retry RPC after error@1531700105) -> (continuation of Deferred@408086754 after retry RPC after error@2099534265) -> (continuation of Deferred@1461196945 after retry RPC after error@1405873744)))
21:03:35.960 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1484244187(state=PENDING, result=null, callback=(continuation of Deferred@30749985 after retry RPC after error@1126640954) -> (continuation of Deferred@1814193286 after retry RPC after error@177546437) -> (continuation of Deferred@574715019 after retry RPC after error@1492501252) -> (continuation of Deferred@663521992 after retry RPC after error@68952560) -> (continuation of Deferred@768669908 after retry RPC after error@1531700105) -> (continuation of Deferred@408086754 after retry RPC after error@2099534265) -> (continuation of Deferred@1461196945 after retry RPC after error@1405873744), errback=(continuation of Deferred@30749985 after retry RPC after error@1126640954) -> (continuation of Deferred@1814193286 after retry RPC after error@177546437) -> (continuation of Deferred@574715019 after retry RPC after error@1492501252) -> (continuation of Deferred@663521992 after retry RPC after error@68952560) -> (continuation of Deferred@768669908 after retry RPC after error@1531700105) -> (continuation of Deferred@408086754 after retry RPC after error@2099534265) -> (continuation of Deferred@1461196945 after retry RPC after error@1405873744)))
21:03:35.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:35.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:35.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:35.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:35.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:35.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:36.044701 4161 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:36.044893 4161 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:36.045550 4161 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:36.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:36.046562 3320 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:03:36.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:36.047732 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:36.047966 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:36.048115 3320 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:36.048440 4161 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:36.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.159 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1018119806(state=PENDING, result=null, callback=(continuation of Deferred@506330253 after retry RPC after error@52344987) -> (continuation of Deferred@934699950 after retry RPC after error@1911312272) -> (continuation of Deferred@13896884 after retry RPC after error@1987622485) -> (continuation of Deferred@1918304295 after retry RPC after error@1632911666) -> (continuation of Deferred@1095061504 after retry RPC after error@1916574819) -> (continuation of Deferred@941685819 after retry RPC after error@2076080092) -> (continuation of Deferred@277937659 after retry RPC after error@114926606), errback=(continuation of Deferred@506330253 after retry RPC after error@52344987) -> (continuation of Deferred@934699950 after retry RPC after error@1911312272) -> (continuation of Deferred@13896884 after retry RPC after error@1987622485) -> (continuation of Deferred@1918304295 after retry RPC after error@1632911666) -> (continuation of Deferred@1095061504 after retry RPC after error@1916574819) -> (continuation of Deferred@941685819 after retry RPC after error@2076080092) -> (continuation of Deferred@277937659 after retry RPC after error@114926606)))
21:03:36.160 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1018119806(state=PENDING, result=null, callback=(continuation of Deferred@506330253 after retry RPC after error@52344987) -> (continuation of Deferred@934699950 after retry RPC after error@1911312272) -> (continuation of Deferred@13896884 after retry RPC after error@1987622485) -> (continuation of Deferred@1918304295 after retry RPC after error@1632911666) -> (continuation of Deferred@1095061504 after retry RPC after error@1916574819) -> (continuation of Deferred@941685819 after retry RPC after error@2076080092) -> (continuation of Deferred@277937659 after retry RPC after error@114926606), errback=(continuation of Deferred@506330253 after retry RPC after error@52344987) -> (continuation of Deferred@934699950 after retry RPC after error@1911312272) -> (continuation of Deferred@13896884 after retry RPC after error@1987622485) -> (continuation of Deferred@1918304295 after retry RPC after error@1632911666) -> (continuation of Deferred@1095061504 after retry RPC after error@1916574819) -> (continuation of Deferred@941685819 after retry RPC after error@2076080092) -> (continuation of Deferred@277937659 after retry RPC after error@114926606)))
21:03:36.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.420 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1682923757(state=PENDING, result=null, callback=(continuation of Deferred@1003047448 after retry RPC after error@732947435) -> (continuation of Deferred@320385092 after retry RPC after error@214954690) -> (continuation of Deferred@2108158153 after retry RPC after error@692319277) -> (continuation of Deferred@2131753937 after retry RPC after error@212164368) -> (continuation of Deferred@1321650165 after retry RPC after error@2014490780) -> (continuation of Deferred@2017664316 after retry RPC after error@1580761339) -> (continuation of Deferred@649661862 after retry RPC after error@1271062461), errback=(continuation of Deferred@1003047448 after retry RPC after error@732947435) -> (continuation of Deferred@320385092 after retry RPC after error@214954690) -> (continuation of Deferred@2108158153 after retry RPC after error@692319277) -> (continuation of Deferred@2131753937 after retry RPC after error@212164368) -> (continuation of Deferred@1321650165 after retry RPC after error@2014490780) -> (continuation of Deferred@2017664316 after retry RPC after error@1580761339) -> (continuation of Deferred@649661862 after retry RPC after error@1271062461)))
21:03:36.420 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1682923757(state=PENDING, result=null, callback=(continuation of Deferred@1003047448 after retry RPC after error@732947435) -> (continuation of Deferred@320385092 after retry RPC after error@214954690) -> (continuation of Deferred@2108158153 after retry RPC after error@692319277) -> (continuation of Deferred@2131753937 after retry RPC after error@212164368) -> (continuation of Deferred@1321650165 after retry RPC after error@2014490780) -> (continuation of Deferred@2017664316 after retry RPC after error@1580761339) -> (continuation of Deferred@649661862 after retry RPC after error@1271062461), errback=(continuation of Deferred@1003047448 after retry RPC after error@732947435) -> (continuation of Deferred@320385092 after retry RPC after error@214954690) -> (continuation of Deferred@2108158153 after retry RPC after error@692319277) -> (continuation of Deferred@2131753937 after retry RPC after error@212164368) -> (continuation of Deferred@1321650165 after retry RPC after error@2014490780) -> (continuation of Deferred@2017664316 after retry RPC after error@1580761339) -> (continuation of Deferred@649661862 after retry RPC after error@1271062461)))
21:03:36.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.457 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.457 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.620 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@87911763(state=PENDING, result=null, callback=(continuation of Deferred@1716348023 after retry RPC after error@993948315) -> (continuation of Deferred@631410805 after retry RPC after error@1268459111) -> (continuation of Deferred@1182305727 after retry RPC after error@236619073) -> (continuation of Deferred@1066044879 after retry RPC after error@758135879) -> (continuation of Deferred@799496491 after retry RPC after error@1976430867) -> (continuation of Deferred@507707024 after retry RPC after error@842808091), errback=(continuation of Deferred@1716348023 after retry RPC after error@993948315) -> (continuation of Deferred@631410805 after retry RPC after error@1268459111) -> (continuation of Deferred@1182305727 after retry RPC after error@236619073) -> (continuation of Deferred@1066044879 after retry RPC after error@758135879) -> (continuation of Deferred@799496491 after retry RPC after error@1976430867) -> (continuation of Deferred@507707024 after retry RPC after error@842808091)))
21:03:36.620 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@87911763(state=PENDING, result=null, callback=(continuation of Deferred@1716348023 after retry RPC after error@993948315) -> (continuation of Deferred@631410805 after retry RPC after error@1268459111) -> (continuation of Deferred@1182305727 after retry RPC after error@236619073) -> (continuation of Deferred@1066044879 after retry RPC after error@758135879) -> (continuation of Deferred@799496491 after retry RPC after error@1976430867) -> (continuation of Deferred@507707024 after retry RPC after error@842808091), errback=(continuation of Deferred@1716348023 after retry RPC after error@993948315) -> (continuation of Deferred@631410805 after retry RPC after error@1268459111) -> (continuation of Deferred@1182305727 after retry RPC after error@236619073) -> (continuation of Deferred@1066044879 after retry RPC after error@758135879) -> (continuation of Deferred@799496491 after retry RPC after error@1976430867) -> (continuation of Deferred@507707024 after retry RPC after error@842808091)))
21:03:36.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.658 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.860 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@214364456(state=PENDING, result=null, callback=(continuation of Deferred@1176635078 after retry RPC after error@990312620) -> (continuation of Deferred@939446738 after retry RPC after error@201996724) -> (continuation of Deferred@1514674885 after retry RPC after error@1443318456) -> (continuation of Deferred@389439979 after retry RPC after error@1252797728) -> (continuation of Deferred@1344837762 after retry RPC after error@1016486581) -> (continuation of Deferred@411786741 after retry RPC after error@870692585) -> (continuation of Deferred@664650911 after retry RPC after error@693137707), errback=(continuation of Deferred@1176635078 after retry RPC after error@990312620) -> (continuation of Deferred@939446738 after retry RPC after error@201996724) -> (continuation of Deferred@1514674885 after retry RPC after error@1443318456) -> (continuation of Deferred@389439979 after retry RPC after error@1252797728) -> (continuation of Deferred@1344837762 after retry RPC after error@1016486581) -> (continuation of Deferred@411786741 after retry RPC after error@870692585) -> (continuation of Deferred@664650911 after retry RPC after error@693137707)))
21:03:36.860 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@214364456(state=PENDING, result=null, callback=(continuation of Deferred@1176635078 after retry RPC after error@990312620) -> (continuation of Deferred@939446738 after retry RPC after error@201996724) -> (continuation of Deferred@1514674885 after retry RPC after error@1443318456) -> (continuation of Deferred@389439979 after retry RPC after error@1252797728) -> (continuation of Deferred@1344837762 after retry RPC after error@1016486581) -> (continuation of Deferred@411786741 after retry RPC after error@870692585) -> (continuation of Deferred@664650911 after retry RPC after error@693137707), errback=(continuation of Deferred@1176635078 after retry RPC after error@990312620) -> (continuation of Deferred@939446738 after retry RPC after error@201996724) -> (continuation of Deferred@1514674885 after retry RPC after error@1443318456) -> (continuation of Deferred@389439979 after retry RPC after error@1252797728) -> (continuation of Deferred@1344837762 after retry RPC after error@1016486581) -> (continuation of Deferred@411786741 after retry RPC after error@870692585) -> (continuation of Deferred@664650911 after retry RPC after error@693137707)))
21:03:36.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.939 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:36.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:36.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:36.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:36.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:36.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.138 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@375496353(state=PENDING, result=null, callback=(continuation of Deferred@120352703 after retry RPC after error@1227314095) -> (continuation of Deferred@1576085731 after retry RPC after error@1199513962) -> (continuation of Deferred@592113581 after retry RPC after error@1342405184) -> (continuation of Deferred@73524883 after retry RPC after error@984343300) -> (continuation of Deferred@1323844731 after retry RPC after error@99808924) -> (continuation of Deferred@246209999 after retry RPC after error@1811590365) -> (continuation of Deferred@1740498501 after retry RPC after error@1969863224), errback=(continuation of Deferred@120352703 after retry RPC after error@1227314095) -> (continuation of Deferred@1576085731 after retry RPC after error@1199513962) -> (continuation of Deferred@592113581 after retry RPC after error@1342405184) -> (continuation of Deferred@73524883 after retry RPC after error@984343300) -> (continuation of Deferred@1323844731 after retry RPC after error@99808924) -> (continuation of Deferred@246209999 after retry RPC after error@1811590365) -> (continuation of Deferred@1740498501 after retry RPC after error@1969863224)))
21:03:37.138 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:37.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.199 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.400 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1610831198(state=PENDING, result=null, callback=(continuation of Deferred@1811383499 after retry RPC after error@1382036133) -> (continuation of Deferred@890230775 after retry RPC after error@1339874733) -> (continuation of Deferred@416995649 after retry RPC after error@1274815973) -> (continuation of Deferred@520962 after retry RPC after error@698316570) -> (continuation of Deferred@1740743609 after retry RPC after error@1303929257) -> (continuation of Deferred@1914180863 after retry RPC after error@544053761) -> (continuation of Deferred@1247488478 after retry RPC after error@1147186197), errback=(continuation of Deferred@1811383499 after retry RPC after error@1382036133) -> (continuation of Deferred@890230775 after retry RPC after error@1339874733) -> (continuation of Deferred@416995649 after retry RPC after error@1274815973) -> (continuation of Deferred@520962 after retry RPC after error@698316570) -> (continuation of Deferred@1740743609 after retry RPC after error@1303929257) -> (continuation of Deferred@1914180863 after retry RPC after error@544053761) -> (continuation of Deferred@1247488478 after retry RPC after error@1147186197)))
21:03:37.400 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1610831198(state=PENDING, result=null, callback=(continuation of Deferred@1811383499 after retry RPC after error@1382036133) -> (continuation of Deferred@890230775 after retry RPC after error@1339874733) -> (continuation of Deferred@416995649 after retry RPC after error@1274815973) -> (continuation of Deferred@520962 after retry RPC after error@698316570) -> (continuation of Deferred@1740743609 after retry RPC after error@1303929257) -> (continuation of Deferred@1914180863 after retry RPC after error@544053761) -> (continuation of Deferred@1247488478 after retry RPC after error@1147186197), errback=(continuation of Deferred@1811383499 after retry RPC after error@1382036133) -> (continuation of Deferred@890230775 after retry RPC after error@1339874733) -> (continuation of Deferred@416995649 after retry RPC after error@1274815973) -> (continuation of Deferred@520962 after retry RPC after error@698316570) -> (continuation of Deferred@1740743609 after retry RPC after error@1303929257) -> (continuation of Deferred@1914180863 after retry RPC after error@544053761) -> (continuation of Deferred@1247488478 after retry RPC after error@1147186197)))
21:03:37.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.439 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.439 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.460 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.660 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@231997838(state=PENDING, result=null, callback=(continuation of Deferred@1452913961 after retry RPC after error@1097463144) -> (continuation of Deferred@948799627 after retry RPC after error@360195685) -> (continuation of Deferred@248768798 after retry RPC after error@10456857) -> (continuation of Deferred@1409455697 after retry RPC after error@10344890) -> (continuation of Deferred@286739374 after retry RPC after error@1829566930) -> (continuation of Deferred@771512892 after retry RPC after error@1136789040) -> (continuation of Deferred@1755545565 after retry RPC after error@1651235186), errback=(continuation of Deferred@1452913961 after retry RPC after error@1097463144) -> (continuation of Deferred@948799627 after retry RPC after error@360195685) -> (continuation of Deferred@248768798 after retry RPC after error@10456857) -> (continuation of Deferred@1409455697 after retry RPC after error@10344890) -> (continuation of Deferred@286739374 after retry RPC after error@1829566930) -> (continuation of Deferred@771512892 after retry RPC after error@1136789040) -> (continuation of Deferred@1755545565 after retry RPC after error@1651235186)))
21:03:37.660 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@231997838(state=PENDING, result=null, callback=(continuation of Deferred@1452913961 after retry RPC after error@1097463144) -> (continuation of Deferred@948799627 after retry RPC after error@360195685) -> (continuation of Deferred@248768798 after retry RPC after error@10456857) -> (continuation of Deferred@1409455697 after retry RPC after error@10344890) -> (continuation of Deferred@286739374 after retry RPC after error@1829566930) -> (continuation of Deferred@771512892 after retry RPC after error@1136789040) -> (continuation of Deferred@1755545565 after retry RPC after error@1651235186), errback=(continuation of Deferred@1452913961 after retry RPC after error@1097463144) -> (continuation of Deferred@948799627 after retry RPC after error@360195685) -> (continuation of Deferred@248768798 after retry RPC after error@10456857) -> (continuation of Deferred@1409455697 after retry RPC after error@10344890) -> (continuation of Deferred@286739374 after retry RPC after error@1829566930) -> (continuation of Deferred@771512892 after retry RPC after error@1136789040) -> (continuation of Deferred@1755545565 after retry RPC after error@1651235186)))
21:03:37.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:37.661016 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 87) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:37.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:37.663349 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 87) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:37.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:37.664721 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:03:37.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.679 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.679 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.698 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.738 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.879 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1678364187(state=PENDING, result=null, callback=(continuation of Deferred@153315059 after retry RPC after error@1309973097) -> (continuation of Deferred@618461712 after retry RPC after error@235897222) -> (continuation of Deferred@470379082 after retry RPC after error@1990491248) -> (continuation of Deferred@904594638 after retry RPC after error@1201320004) -> (continuation of Deferred@278321607 after retry RPC after error@1166303321) -> (continuation of Deferred@673501523 after retry RPC after error@984505475) -> (continuation of Deferred@475067120 after retry RPC after error@2058411915), errback=(continuation of Deferred@153315059 after retry RPC after error@1309973097) -> (continuation of Deferred@618461712 after retry RPC after error@235897222) -> (continuation of Deferred@470379082 after retry RPC after error@1990491248) -> (continuation of Deferred@904594638 after retry RPC after error@1201320004) -> (continuation of Deferred@278321607 after retry RPC after error@1166303321) -> (continuation of Deferred@673501523 after retry RPC after error@984505475) -> (continuation of Deferred@475067120 after retry RPC after error@2058411915)))
21:03:37.880 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1678364187(state=PENDING, result=null, callback=(continuation of Deferred@153315059 after retry RPC after error@1309973097) -> (continuation of Deferred@618461712 after retry RPC after error@235897222) -> (continuation of Deferred@470379082 after retry RPC after error@1990491248) -> (continuation of Deferred@904594638 after retry RPC after error@1201320004) -> (continuation of Deferred@278321607 after retry RPC after error@1166303321) -> (continuation of Deferred@673501523 after retry RPC after error@984505475) -> (continuation of Deferred@475067120 after retry RPC after error@2058411915), errback=(continuation of Deferred@153315059 after retry RPC after error@1309973097) -> (continuation of Deferred@618461712 after retry RPC after error@235897222) -> (continuation of Deferred@470379082 after retry RPC after error@1990491248) -> (continuation of Deferred@904594638 after retry RPC after error@1201320004) -> (continuation of Deferred@278321607 after retry RPC after error@1166303321) -> (continuation of Deferred@673501523 after retry RPC after error@984505475) -> (continuation of Deferred@475067120 after retry RPC after error@2058411915)))
21:03:37.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:37.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:37.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:37.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:37.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:37.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.037 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.037 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.038 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.157 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1988586811(state=PENDING, result=null, callback=(continuation of Deferred@166417520 after retry RPC after error@417469883) -> (continuation of Deferred@1951598465 after retry RPC after error@972051077) -> (continuation of Deferred@1793837343 after retry RPC after error@1655457332) -> (continuation of Deferred@237972207 after retry RPC after error@904930773) -> (continuation of Deferred@23572805 after retry RPC after error@33213115) -> (continuation of Deferred@1987436601 after retry RPC after error@1109372335) -> (continuation of Deferred@2127483184 after retry RPC after error@699818816), errback=(continuation of Deferred@166417520 after retry RPC after error@417469883) -> (continuation of Deferred@1951598465 after retry RPC after error@972051077) -> (continuation of Deferred@1793837343 after retry RPC after error@1655457332) -> (continuation of Deferred@237972207 after retry RPC after error@904930773) -> (continuation of Deferred@23572805 after retry RPC after error@33213115) -> (continuation of Deferred@1987436601 after retry RPC after error@1109372335) -> (continuation of Deferred@2127483184 after retry RPC after error@699818816)))
21:03:38.158 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:38.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.178 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.258 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.419 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.420 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@899494694(state=PENDING, result=null, callback=(continuation of Deferred@816291984 after retry RPC after error@1624458593) -> (continuation of Deferred@609322604 after retry RPC after error@2013889987) -> (continuation of Deferred@1973881202 after retry RPC after error@262517482) -> (continuation of Deferred@661596052 after retry RPC after error@326932775) -> (continuation of Deferred@1591602851 after retry RPC after error@2085434736) -> (continuation of Deferred@574684906 after retry RPC after error@1426259492) -> (continuation of Deferred@1679534801 after retry RPC after error@1902699927), errback=(continuation of Deferred@816291984 after retry RPC after error@1624458593) -> (continuation of Deferred@609322604 after retry RPC after error@2013889987) -> (continuation of Deferred@1973881202 after retry RPC after error@262517482) -> (continuation of Deferred@661596052 after retry RPC after error@326932775) -> (continuation of Deferred@1591602851 after retry RPC after error@2085434736) -> (continuation of Deferred@574684906 after retry RPC after error@1426259492) -> (continuation of Deferred@1679534801 after retry RPC after error@1902699927)))
21:03:38.421 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@899494694(state=PENDING, result=null, callback=(continuation of Deferred@816291984 after retry RPC after error@1624458593) -> (continuation of Deferred@609322604 after retry RPC after error@2013889987) -> (continuation of Deferred@1973881202 after retry RPC after error@262517482) -> (continuation of Deferred@661596052 after retry RPC after error@326932775) -> (continuation of Deferred@1591602851 after retry RPC after error@2085434736) -> (continuation of Deferred@574684906 after retry RPC after error@1426259492) -> (continuation of Deferred@1679534801 after retry RPC after error@1902699927), errback=(continuation of Deferred@816291984 after retry RPC after error@1624458593) -> (continuation of Deferred@609322604 after retry RPC after error@2013889987) -> (continuation of Deferred@1973881202 after retry RPC after error@262517482) -> (continuation of Deferred@661596052 after retry RPC after error@326932775) -> (continuation of Deferred@1591602851 after retry RPC after error@2085434736) -> (continuation of Deferred@574684906 after retry RPC after error@1426259492) -> (continuation of Deferred@1679534801 after retry RPC after error@1902699927)))
21:03:38.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.459 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.539 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.539 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.540 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.639 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@862226222(state=PENDING, result=null, callback=(continuation of Deferred@831141213 after retry RPC after error@470781693) -> (continuation of Deferred@1187864192 after retry RPC after error@1359473792) -> (continuation of Deferred@826743584 after retry RPC after error@1373569939) -> (continuation of Deferred@2102034038 after retry RPC after error@323485363) -> (continuation of Deferred@449870304 after retry RPC after error@859447986) -> (continuation of Deferred@497898349 after retry RPC after error@825459725) -> (continuation of Deferred@683009202 after retry RPC after error@1249406116) -> (continuation of Deferred@1027172541 after retry RPC after error@277319931), errback=(continuation of Deferred@831141213 after retry RPC after error@470781693) -> (continuation of Deferred@1187864192 after retry RPC after error@1359473792) -> (continuation of Deferred@826743584 after retry RPC after error@1373569939) -> (continuation of Deferred@2102034038 after retry RPC after error@323485363) -> (continuation of Deferred@449870304 after retry RPC after error@859447986) -> (continuation of Deferred@497898349 after retry RPC after error@825459725) -> (continuation of Deferred@683009202 after retry RPC after error@1249406116) -> (continuation of Deferred@1027172541 after retry RPC after error@277319931)))
21:03:38.640 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@862226222(state=PENDING, result=null, callback=(continuation of Deferred@831141213 after retry RPC after error@470781693) -> (continuation of Deferred@1187864192 after retry RPC after error@1359473792) -> (continuation of Deferred@826743584 after retry RPC after error@1373569939) -> (continuation of Deferred@2102034038 after retry RPC after error@323485363) -> (continuation of Deferred@449870304 after retry RPC after error@859447986) -> (continuation of Deferred@497898349 after retry RPC after error@825459725) -> (continuation of Deferred@683009202 after retry RPC after error@1249406116) -> (continuation of Deferred@1027172541 after retry RPC after error@277319931), errback=(continuation of Deferred@831141213 after retry RPC after error@470781693) -> (continuation of Deferred@1187864192 after retry RPC after error@1359473792) -> (continuation of Deferred@826743584 after retry RPC after error@1373569939) -> (continuation of Deferred@2102034038 after retry RPC after error@323485363) -> (continuation of Deferred@449870304 after retry RPC after error@859447986) -> (continuation of Deferred@497898349 after retry RPC after error@825459725) -> (continuation of Deferred@683009202 after retry RPC after error@1249406116) -> (continuation of Deferred@1027172541 after retry RPC after error@277319931)))
21:03:38.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.680 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.757 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.757 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.758 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.860 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@337257726(state=PENDING, result=null, callback=(continuation of Deferred@1960390984 after retry RPC after error@579754896) -> (continuation of Deferred@599070752 after retry RPC after error@1724773756) -> (continuation of Deferred@1529076636 after retry RPC after error@186730931) -> (continuation of Deferred@14562543 after retry RPC after error@162613149) -> (continuation of Deferred@1284211212 after retry RPC after error@1298540287) -> (continuation of Deferred@1022237424 after retry RPC after error@1071656799) -> (continuation of Deferred@803950925 after retry RPC after error@1405537169), errback=(continuation of Deferred@1960390984 after retry RPC after error@579754896) -> (continuation of Deferred@599070752 after retry RPC after error@1724773756) -> (continuation of Deferred@1529076636 after retry RPC after error@186730931) -> (continuation of Deferred@14562543 after retry RPC after error@162613149) -> (continuation of Deferred@1284211212 after retry RPC after error@1298540287) -> (continuation of Deferred@1022237424 after retry RPC after error@1071656799) -> (continuation of Deferred@803950925 after retry RPC after error@1405537169)))
21:03:38.860 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@337257726(state=PENDING, result=null, callback=(continuation of Deferred@1960390984 after retry RPC after error@579754896) -> (continuation of Deferred@599070752 after retry RPC after error@1724773756) -> (continuation of Deferred@1529076636 after retry RPC after error@186730931) -> (continuation of Deferred@14562543 after retry RPC after error@162613149) -> (continuation of Deferred@1284211212 after retry RPC after error@1298540287) -> (continuation of Deferred@1022237424 after retry RPC after error@1071656799) -> (continuation of Deferred@803950925 after retry RPC after error@1405537169), errback=(continuation of Deferred@1960390984 after retry RPC after error@579754896) -> (continuation of Deferred@599070752 after retry RPC after error@1724773756) -> (continuation of Deferred@1529076636 after retry RPC after error@186730931) -> (continuation of Deferred@14562543 after retry RPC after error@162613149) -> (continuation of Deferred@1284211212 after retry RPC after error@1298540287) -> (continuation of Deferred@1022237424 after retry RPC after error@1071656799) -> (continuation of Deferred@803950925 after retry RPC after error@1405537169)))
21:03:38.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:38.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:38.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:38.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:38.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:38.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.039 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@883812573(state=PENDING, result=null, callback=(continuation of Deferred@544444672 after retry RPC after error@1542937678) -> (continuation of Deferred@28568928 after retry RPC after error@1518540427) -> (continuation of Deferred@1724151626 after retry RPC after error@120607472) -> (continuation of Deferred@435578779 after retry RPC after error@1535319392) -> (continuation of Deferred@507006845 after retry RPC after error@454699416) -> (continuation of Deferred@830103519 after retry RPC after error@1917926541) -> (continuation of Deferred@776914461 after retry RPC after error@1691511973), errback=(continuation of Deferred@544444672 after retry RPC after error@1542937678) -> (continuation of Deferred@28568928 after retry RPC after error@1518540427) -> (continuation of Deferred@1724151626 after retry RPC after error@120607472) -> (continuation of Deferred@435578779 after retry RPC after error@1535319392) -> (continuation of Deferred@507006845 after retry RPC after error@454699416) -> (continuation of Deferred@830103519 after retry RPC after error@1917926541) -> (continuation of Deferred@776914461 after retry RPC after error@1691511973)))
21:03:39.040 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@883812573(state=PENDING, result=null, callback=(continuation of Deferred@544444672 after retry RPC after error@1542937678) -> (continuation of Deferred@28568928 after retry RPC after error@1518540427) -> (continuation of Deferred@1724151626 after retry RPC after error@120607472) -> (continuation of Deferred@435578779 after retry RPC after error@1535319392) -> (continuation of Deferred@507006845 after retry RPC after error@454699416) -> (continuation of Deferred@830103519 after retry RPC after error@1917926541) -> (continuation of Deferred@776914461 after retry RPC after error@1691511973), errback=(continuation of Deferred@544444672 after retry RPC after error@1542937678) -> (continuation of Deferred@28568928 after retry RPC after error@1518540427) -> (continuation of Deferred@1724151626 after retry RPC after error@120607472) -> (continuation of Deferred@435578779 after retry RPC after error@1535319392) -> (continuation of Deferred@507006845 after retry RPC after error@454699416) -> (continuation of Deferred@830103519 after retry RPC after error@1917926541) -> (continuation of Deferred@776914461 after retry RPC after error@1691511973)))
21:03:39.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.059 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.060 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.299 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1785623497(state=PENDING, result=null, callback=(continuation of Deferred@1005569008 after retry RPC after error@830788961) -> (continuation of Deferred@1670366366 after retry RPC after error@2053505821) -> (continuation of Deferred@1070841826 after retry RPC after error@686508708) -> (continuation of Deferred@895340563 after retry RPC after error@2137604942) -> (continuation of Deferred@1575473134 after retry RPC after error@791347356) -> (continuation of Deferred@1949138978 after retry RPC after error@1097584346) -> (continuation of Deferred@571531686 after retry RPC after error@1439185560) -> (continuation of Deferred@1308396691 after retry RPC after error@1392412392), errback=(continuation of Deferred@1005569008 after retry RPC after error@830788961) -> (continuation of Deferred@1670366366 after retry RPC after error@2053505821) -> (continuation of Deferred@1070841826 after retry RPC after error@686508708) -> (continuation of Deferred@895340563 after retry RPC after error@2137604942) -> (continuation of Deferred@1575473134 after retry RPC after error@791347356) -> (continuation of Deferred@1949138978 after retry RPC after error@1097584346) -> (continuation of Deferred@571531686 after retry RPC after error@1439185560) -> (continuation of Deferred@1308396691 after retry RPC after error@1392412392)))
21:03:39.300 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1785623497(state=PENDING, result=null, callback=(continuation of Deferred@1005569008 after retry RPC after error@830788961) -> (continuation of Deferred@1670366366 after retry RPC after error@2053505821) -> (continuation of Deferred@1070841826 after retry RPC after error@686508708) -> (continuation of Deferred@895340563 after retry RPC after error@2137604942) -> (continuation of Deferred@1575473134 after retry RPC after error@791347356) -> (continuation of Deferred@1949138978 after retry RPC after error@1097584346) -> (continuation of Deferred@571531686 after retry RPC after error@1439185560) -> (continuation of Deferred@1308396691 after retry RPC after error@1392412392), errback=(continuation of Deferred@1005569008 after retry RPC after error@830788961) -> (continuation of Deferred@1670366366 after retry RPC after error@2053505821) -> (continuation of Deferred@1070841826 after retry RPC after error@686508708) -> (continuation of Deferred@895340563 after retry RPC after error@2137604942) -> (continuation of Deferred@1575473134 after retry RPC after error@791347356) -> (continuation of Deferred@1949138978 after retry RPC after error@1097584346) -> (continuation of Deferred@571531686 after retry RPC after error@1439185560) -> (continuation of Deferred@1308396691 after retry RPC after error@1392412392)))
21:03:39.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.317 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.379 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.500 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@175414440(state=PENDING, result=null, callback=(continuation of Deferred@370056755 after retry RPC after error@1597121140) -> (continuation of Deferred@790896027 after retry RPC after error@18564363) -> (continuation of Deferred@1661385757 after retry RPC after error@761856907) -> (continuation of Deferred@573447032 after retry RPC after error@2043031851) -> (continuation of Deferred@804691034 after retry RPC after error@1248245620) -> (continuation of Deferred@415334555 after retry RPC after error@1346811329), errback=(continuation of Deferred@370056755 after retry RPC after error@1597121140) -> (continuation of Deferred@790896027 after retry RPC after error@18564363) -> (continuation of Deferred@1661385757 after retry RPC after error@761856907) -> (continuation of Deferred@573447032 after retry RPC after error@2043031851) -> (continuation of Deferred@804691034 after retry RPC after error@1248245620) -> (continuation of Deferred@415334555 after retry RPC after error@1346811329)))
21:03:39.501 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@175414440(state=PENDING, result=null, callback=(continuation of Deferred@370056755 after retry RPC after error@1597121140) -> (continuation of Deferred@790896027 after retry RPC after error@18564363) -> (continuation of Deferred@1661385757 after retry RPC after error@761856907) -> (continuation of Deferred@573447032 after retry RPC after error@2043031851) -> (continuation of Deferred@804691034 after retry RPC after error@1248245620) -> (continuation of Deferred@415334555 after retry RPC after error@1346811329), errback=(continuation of Deferred@370056755 after retry RPC after error@1597121140) -> (continuation of Deferred@790896027 after retry RPC after error@18564363) -> (continuation of Deferred@1661385757 after retry RPC after error@761856907) -> (continuation of Deferred@573447032 after retry RPC after error@2043031851) -> (continuation of Deferred@804691034 after retry RPC after error@1248245620) -> (continuation of Deferred@415334555 after retry RPC after error@1346811329)))
21:03:39.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.519 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.680 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@27790307(state=PENDING, result=null, callback=(continuation of Deferred@809403842 after retry RPC after error@2044521933) -> (continuation of Deferred@685120065 after retry RPC after error@1776352171) -> (continuation of Deferred@766796736 after retry RPC after error@857227582) -> (continuation of Deferred@730087694 after retry RPC after error@1163385462) -> (continuation of Deferred@1274219391 after retry RPC after error@982758030) -> (continuation of Deferred@857086179 after retry RPC after error@1223286941), errback=(continuation of Deferred@809403842 after retry RPC after error@2044521933) -> (continuation of Deferred@685120065 after retry RPC after error@1776352171) -> (continuation of Deferred@766796736 after retry RPC after error@857227582) -> (continuation of Deferred@730087694 after retry RPC after error@1163385462) -> (continuation of Deferred@1274219391 after retry RPC after error@982758030) -> (continuation of Deferred@857086179 after retry RPC after error@1223286941)))
21:03:39.680 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@27790307(state=PENDING, result=null, callback=(continuation of Deferred@809403842 after retry RPC after error@2044521933) -> (continuation of Deferred@685120065 after retry RPC after error@1776352171) -> (continuation of Deferred@766796736 after retry RPC after error@857227582) -> (continuation of Deferred@730087694 after retry RPC after error@1163385462) -> (continuation of Deferred@1274219391 after retry RPC after error@982758030) -> (continuation of Deferred@857086179 after retry RPC after error@1223286941), errback=(continuation of Deferred@809403842 after retry RPC after error@2044521933) -> (continuation of Deferred@685120065 after retry RPC after error@1776352171) -> (continuation of Deferred@766796736 after retry RPC after error@857227582) -> (continuation of Deferred@730087694 after retry RPC after error@1163385462) -> (continuation of Deferred@1274219391 after retry RPC after error@982758030) -> (continuation of Deferred@857086179 after retry RPC after error@1223286941)))
21:03:39.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.778 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.859 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.860 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@358024002(state=PENDING, result=null, callback=(continuation of Deferred@1083628903 after retry RPC after error@1822910580) -> (continuation of Deferred@1091567138 after retry RPC after error@1819256914) -> (continuation of Deferred@451165533 after retry RPC after error@1297547965) -> (continuation of Deferred@1939278811 after retry RPC after error@1655374019) -> (continuation of Deferred@1495189278 after retry RPC after error@740910186) -> (continuation of Deferred@1739291623 after retry RPC after error@1572651721), errback=(continuation of Deferred@1083628903 after retry RPC after error@1822910580) -> (continuation of Deferred@1091567138 after retry RPC after error@1819256914) -> (continuation of Deferred@451165533 after retry RPC after error@1297547965) -> (continuation of Deferred@1939278811 after retry RPC after error@1655374019) -> (continuation of Deferred@1495189278 after retry RPC after error@740910186) -> (continuation of Deferred@1739291623 after retry RPC after error@1572651721)))
21:03:39.860 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@358024002(state=PENDING, result=null, callback=(continuation of Deferred@1083628903 after retry RPC after error@1822910580) -> (continuation of Deferred@1091567138 after retry RPC after error@1819256914) -> (continuation of Deferred@451165533 after retry RPC after error@1297547965) -> (continuation of Deferred@1939278811 after retry RPC after error@1655374019) -> (continuation of Deferred@1495189278 after retry RPC after error@740910186) -> (continuation of Deferred@1739291623 after retry RPC after error@1572651721), errback=(continuation of Deferred@1083628903 after retry RPC after error@1822910580) -> (continuation of Deferred@1091567138 after retry RPC after error@1819256914) -> (continuation of Deferred@451165533 after retry RPC after error@1297547965) -> (continuation of Deferred@1939278811 after retry RPC after error@1655374019) -> (continuation of Deferred@1495189278 after retry RPC after error@740910186) -> (continuation of Deferred@1739291623 after retry RPC after error@1572651721)))
21:03:39.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.937 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:39.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:39.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:39.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:39.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:39.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.058 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.099 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.100 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.100 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@839377354(state=PENDING, result=null, callback=(continuation of Deferred@1917443958 after retry RPC after error@1399830489) -> (continuation of Deferred@440003995 after retry RPC after error@389895847) -> (continuation of Deferred@733186393 after retry RPC after error@97264265) -> (continuation of Deferred@1347179166 after retry RPC after error@758759006) -> (continuation of Deferred@444011489 after retry RPC after error@2118885964) -> (continuation of Deferred@1285670841 after retry RPC after error@738876221) -> (continuation of Deferred@239123782 after retry RPC after error@767174400), errback=(continuation of Deferred@1917443958 after retry RPC after error@1399830489) -> (continuation of Deferred@440003995 after retry RPC after error@389895847) -> (continuation of Deferred@733186393 after retry RPC after error@97264265) -> (continuation of Deferred@1347179166 after retry RPC after error@758759006) -> (continuation of Deferred@444011489 after retry RPC after error@2118885964) -> (continuation of Deferred@1285670841 after retry RPC after error@738876221) -> (continuation of Deferred@239123782 after retry RPC after error@767174400)))
21:03:40.100 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@839377354(state=PENDING, result=null, callback=(continuation of Deferred@1917443958 after retry RPC after error@1399830489) -> (continuation of Deferred@440003995 after retry RPC after error@389895847) -> (continuation of Deferred@733186393 after retry RPC after error@97264265) -> (continuation of Deferred@1347179166 after retry RPC after error@758759006) -> (continuation of Deferred@444011489 after retry RPC after error@2118885964) -> (continuation of Deferred@1285670841 after retry RPC after error@738876221) -> (continuation of Deferred@239123782 after retry RPC after error@767174400), errback=(continuation of Deferred@1917443958 after retry RPC after error@1399830489) -> (continuation of Deferred@440003995 after retry RPC after error@389895847) -> (continuation of Deferred@733186393 after retry RPC after error@97264265) -> (continuation of Deferred@1347179166 after retry RPC after error@758759006) -> (continuation of Deferred@444011489 after retry RPC after error@2118885964) -> (continuation of Deferred@1285670841 after retry RPC after error@738876221) -> (continuation of Deferred@239123782 after retry RPC after error@767174400)))
21:03:40.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.157 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.157 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.157 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.158 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.338 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.339 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1559869471(state=PENDING, result=null, callback=(continuation of Deferred@202159755 after retry RPC after error@1023141587) -> (continuation of Deferred@481604423 after retry RPC after error@228383118) -> (continuation of Deferred@1682808408 after retry RPC after error@123941244) -> (continuation of Deferred@1047827548 after retry RPC after error@1431268021) -> (continuation of Deferred@1568178458 after retry RPC after error@1133421336) -> (continuation of Deferred@1505892673 after retry RPC after error@2012662286) -> (continuation of Deferred@1855972795 after retry RPC after error@1488919827) -> (continuation of Deferred@301866836 after retry RPC after error@1352208863), errback=(continuation of Deferred@202159755 after retry RPC after error@1023141587) -> (continuation of Deferred@481604423 after retry RPC after error@228383118) -> (continuation of Deferred@1682808408 after retry RPC after error@123941244) -> (continuation of Deferred@1047827548 after retry RPC after error@1431268021) -> (continuation of Deferred@1568178458 after retry RPC after error@1133421336) -> (continuation of Deferred@1505892673 after retry RPC after error@2012662286) -> (continuation of Deferred@1855972795 after retry RPC after error@1488919827) -> (continuation of Deferred@301866836 after retry RPC after error@1352208863)))
21:03:40.339 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1559869471(state=PENDING, result=null, callback=(continuation of Deferred@202159755 after retry RPC after error@1023141587) -> (continuation of Deferred@481604423 after retry RPC after error@228383118) -> (continuation of Deferred@1682808408 after retry RPC after error@123941244) -> (continuation of Deferred@1047827548 after retry RPC after error@1431268021) -> (continuation of Deferred@1568178458 after retry RPC after error@1133421336) -> (continuation of Deferred@1505892673 after retry RPC after error@2012662286) -> (continuation of Deferred@1855972795 after retry RPC after error@1488919827) -> (continuation of Deferred@301866836 after retry RPC after error@1352208863), errback=(continuation of Deferred@202159755 after retry RPC after error@1023141587) -> (continuation of Deferred@481604423 after retry RPC after error@228383118) -> (continuation of Deferred@1682808408 after retry RPC after error@123941244) -> (continuation of Deferred@1047827548 after retry RPC after error@1431268021) -> (continuation of Deferred@1568178458 after retry RPC after error@1133421336) -> (continuation of Deferred@1505892673 after retry RPC after error@2012662286) -> (continuation of Deferred@1855972795 after retry RPC after error@1488919827) -> (continuation of Deferred@301866836 after retry RPC after error@1352208863)))
21:03:40.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.600 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@14698465(state=PENDING, result=null, callback=(continuation of Deferred@258207852 after retry RPC after error@276998653) -> (continuation of Deferred@1934795542 after retry RPC after error@1896820178) -> (continuation of Deferred@1635734287 after retry RPC after error@1681376156) -> (continuation of Deferred@35362335 after retry RPC after error@175262259) -> (continuation of Deferred@294553433 after retry RPC after error@1266650583) -> (continuation of Deferred@352997833 after retry RPC after error@229745289) -> (continuation of Deferred@1751140393 after retry RPC after error@1598452167), errback=(continuation of Deferred@258207852 after retry RPC after error@276998653) -> (continuation of Deferred@1934795542 after retry RPC after error@1896820178) -> (continuation of Deferred@1635734287 after retry RPC after error@1681376156) -> (continuation of Deferred@35362335 after retry RPC after error@175262259) -> (continuation of Deferred@294553433 after retry RPC after error@1266650583) -> (continuation of Deferred@352997833 after retry RPC after error@229745289) -> (continuation of Deferred@1751140393 after retry RPC after error@1598452167)))
21:03:40.600 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@14698465(state=PENDING, result=null, callback=(continuation of Deferred@258207852 after retry RPC after error@276998653) -> (continuation of Deferred@1934795542 after retry RPC after error@1896820178) -> (continuation of Deferred@1635734287 after retry RPC after error@1681376156) -> (continuation of Deferred@35362335 after retry RPC after error@175262259) -> (continuation of Deferred@294553433 after retry RPC after error@1266650583) -> (continuation of Deferred@352997833 after retry RPC after error@229745289) -> (continuation of Deferred@1751140393 after retry RPC after error@1598452167), errback=(continuation of Deferred@258207852 after retry RPC after error@276998653) -> (continuation of Deferred@1934795542 after retry RPC after error@1896820178) -> (continuation of Deferred@1635734287 after retry RPC after error@1681376156) -> (continuation of Deferred@35362335 after retry RPC after error@175262259) -> (continuation of Deferred@294553433 after retry RPC after error@1266650583) -> (continuation of Deferred@352997833 after retry RPC after error@229745289) -> (continuation of Deferred@1751140393 after retry RPC after error@1598452167)))
21:03:40.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.638 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:40.791844 4183 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:40.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:40.792075 4183 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:40.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:40.792744 4183 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:40.794909 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:40.795112 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:40.795212 3320 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:40.795449 4183 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:40.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.799 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1837910608(state=PENDING, result=null, callback=(continuation of Deferred@1998879170 after retry RPC after error@1755854503) -> (continuation of Deferred@2086364064 after retry RPC after error@1930066097) -> (continuation of Deferred@657778043 after retry RPC after error@234943028) -> (continuation of Deferred@702666295 after retry RPC after error@160365642) -> (continuation of Deferred@152166205 after retry RPC after error@8981881) -> (continuation of Deferred@662577969 after retry RPC after error@1500309304) -> (continuation of Deferred@1585656607 after retry RPC after error@380182387), errback=(continuation of Deferred@1998879170 after retry RPC after error@1755854503) -> (continuation of Deferred@2086364064 after retry RPC after error@1930066097) -> (continuation of Deferred@657778043 after retry RPC after error@234943028) -> (continuation of Deferred@702666295 after retry RPC after error@160365642) -> (continuation of Deferred@152166205 after retry RPC after error@8981881) -> (continuation of Deferred@662577969 after retry RPC after error@1500309304) -> (continuation of Deferred@1585656607 after retry RPC after error@380182387)))
21:03:40.800 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1837910608(state=PENDING, result=null, callback=(continuation of Deferred@1998879170 after retry RPC after error@1755854503) -> (continuation of Deferred@2086364064 after retry RPC after error@1930066097) -> (continuation of Deferred@657778043 after retry RPC after error@234943028) -> (continuation of Deferred@702666295 after retry RPC after error@160365642) -> (continuation of Deferred@152166205 after retry RPC after error@8981881) -> (continuation of Deferred@662577969 after retry RPC after error@1500309304) -> (continuation of Deferred@1585656607 after retry RPC after error@380182387), errback=(continuation of Deferred@1998879170 after retry RPC after error@1755854503) -> (continuation of Deferred@2086364064 after retry RPC after error@1930066097) -> (continuation of Deferred@657778043 after retry RPC after error@234943028) -> (continuation of Deferred@702666295 after retry RPC after error@160365642) -> (continuation of Deferred@152166205 after retry RPC after error@8981881) -> (continuation of Deferred@662577969 after retry RPC after error@1500309304) -> (continuation of Deferred@1585656607 after retry RPC after error@380182387)))
21:03:40.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:40.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:40.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:40.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:40.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:40.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.040 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1094466897(state=PENDING, result=null, callback=(continuation of Deferred@1507363926 after retry RPC after error@1509864484) -> (continuation of Deferred@272566296 after retry RPC after error@818510331) -> (continuation of Deferred@770632072 after retry RPC after error@27076485) -> (continuation of Deferred@1770688614 after retry RPC after error@309803533) -> (continuation of Deferred@1217340258 after retry RPC after error@549885177) -> (continuation of Deferred@502413607 after retry RPC after error@914280390) -> (continuation of Deferred@123432536 after retry RPC after error@321798934), errback=(continuation of Deferred@1507363926 after retry RPC after error@1509864484) -> (continuation of Deferred@272566296 after retry RPC after error@818510331) -> (continuation of Deferred@770632072 after retry RPC after error@27076485) -> (continuation of Deferred@1770688614 after retry RPC after error@309803533) -> (continuation of Deferred@1217340258 after retry RPC after error@549885177) -> (continuation of Deferred@502413607 after retry RPC after error@914280390) -> (continuation of Deferred@123432536 after retry RPC after error@321798934)))
21:03:41.040 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1094466897(state=PENDING, result=null, callback=(continuation of Deferred@1507363926 after retry RPC after error@1509864484) -> (continuation of Deferred@272566296 after retry RPC after error@818510331) -> (continuation of Deferred@770632072 after retry RPC after error@27076485) -> (continuation of Deferred@1770688614 after retry RPC after error@309803533) -> (continuation of Deferred@1217340258 after retry RPC after error@549885177) -> (continuation of Deferred@502413607 after retry RPC after error@914280390) -> (continuation of Deferred@123432536 after retry RPC after error@321798934), errback=(continuation of Deferred@1507363926 after retry RPC after error@1509864484) -> (continuation of Deferred@272566296 after retry RPC after error@818510331) -> (continuation of Deferred@770632072 after retry RPC after error@27076485) -> (continuation of Deferred@1770688614 after retry RPC after error@309803533) -> (continuation of Deferred@1217340258 after retry RPC after error@549885177) -> (continuation of Deferred@502413607 after retry RPC after error@914280390) -> (continuation of Deferred@123432536 after retry RPC after error@321798934)))
21:03:41.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.220 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1262010445(state=PENDING, result=null, callback=(continuation of Deferred@1084622100 after retry RPC after error@1902947769) -> (continuation of Deferred@959089147 after retry RPC after error@882707515) -> (continuation of Deferred@1155735833 after retry RPC after error@1349934425) -> (continuation of Deferred@1040081917 after retry RPC after error@1641962669) -> (continuation of Deferred@1021858406 after retry RPC after error@1586908215) -> (continuation of Deferred@272182675 after retry RPC after error@59842573) -> (continuation of Deferred@2098473339 after retry RPC after error@286205210), errback=(continuation of Deferred@1084622100 after retry RPC after error@1902947769) -> (continuation of Deferred@959089147 after retry RPC after error@882707515) -> (continuation of Deferred@1155735833 after retry RPC after error@1349934425) -> (continuation of Deferred@1040081917 after retry RPC after error@1641962669) -> (continuation of Deferred@1021858406 after retry RPC after error@1586908215) -> (continuation of Deferred@272182675 after retry RPC after error@59842573) -> (continuation of Deferred@2098473339 after retry RPC after error@286205210)))
21:03:41.221 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1262010445(state=PENDING, result=null, callback=(continuation of Deferred@1084622100 after retry RPC after error@1902947769) -> (continuation of Deferred@959089147 after retry RPC after error@882707515) -> (continuation of Deferred@1155735833 after retry RPC after error@1349934425) -> (continuation of Deferred@1040081917 after retry RPC after error@1641962669) -> (continuation of Deferred@1021858406 after retry RPC after error@1586908215) -> (continuation of Deferred@272182675 after retry RPC after error@59842573) -> (continuation of Deferred@2098473339 after retry RPC after error@286205210), errback=(continuation of Deferred@1084622100 after retry RPC after error@1902947769) -> (continuation of Deferred@959089147 after retry RPC after error@882707515) -> (continuation of Deferred@1155735833 after retry RPC after error@1349934425) -> (continuation of Deferred@1040081917 after retry RPC after error@1641962669) -> (continuation of Deferred@1021858406 after retry RPC after error@1586908215) -> (continuation of Deferred@272182675 after retry RPC after error@59842573) -> (continuation of Deferred@2098473339 after retry RPC after error@286205210)))
21:03:41.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.298 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.440 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@4508818(state=PENDING, result=null, callback=(continuation of Deferred@61308818 after retry RPC after error@503636979) -> (continuation of Deferred@554372860 after retry RPC after error@126707219) -> (continuation of Deferred@1064227487 after retry RPC after error@114075130) -> (continuation of Deferred@2135504689 after retry RPC after error@328756021) -> (continuation of Deferred@1521199603 after retry RPC after error@959132713) -> (continuation of Deferred@258833538 after retry RPC after error@1610379161), errback=(continuation of Deferred@61308818 after retry RPC after error@503636979) -> (continuation of Deferred@554372860 after retry RPC after error@126707219) -> (continuation of Deferred@1064227487 after retry RPC after error@114075130) -> (continuation of Deferred@2135504689 after retry RPC after error@328756021) -> (continuation of Deferred@1521199603 after retry RPC after error@959132713) -> (continuation of Deferred@258833538 after retry RPC after error@1610379161)))
21:03:41.440 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@4508818(state=PENDING, result=null, callback=(continuation of Deferred@61308818 after retry RPC after error@503636979) -> (continuation of Deferred@554372860 after retry RPC after error@126707219) -> (continuation of Deferred@1064227487 after retry RPC after error@114075130) -> (continuation of Deferred@2135504689 after retry RPC after error@328756021) -> (continuation of Deferred@1521199603 after retry RPC after error@959132713) -> (continuation of Deferred@258833538 after retry RPC after error@1610379161), errback=(continuation of Deferred@61308818 after retry RPC after error@503636979) -> (continuation of Deferred@554372860 after retry RPC after error@126707219) -> (continuation of Deferred@1064227487 after retry RPC after error@114075130) -> (continuation of Deferred@2135504689 after retry RPC after error@328756021) -> (continuation of Deferred@1521199603 after retry RPC after error@959132713) -> (continuation of Deferred@258833538 after retry RPC after error@1610379161)))
21:03:41.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.597 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.597 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.700 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1088694356(state=PENDING, result=null, callback=(continuation of Deferred@2102977504 after retry RPC after error@1879419762) -> (continuation of Deferred@28051970 after retry RPC after error@1617638635) -> (continuation of Deferred@1519815909 after retry RPC after error@127263486) -> (continuation of Deferred@1720189972 after retry RPC after error@1610463298) -> (continuation of Deferred@836953852 after retry RPC after error@432562220) -> (continuation of Deferred@1069969317 after retry RPC after error@223080375) -> (continuation of Deferred@725106148 after retry RPC after error@1907089839), errback=(continuation of Deferred@2102977504 after retry RPC after error@1879419762) -> (continuation of Deferred@28051970 after retry RPC after error@1617638635) -> (continuation of Deferred@1519815909 after retry RPC after error@127263486) -> (continuation of Deferred@1720189972 after retry RPC after error@1610463298) -> (continuation of Deferred@836953852 after retry RPC after error@432562220) -> (continuation of Deferred@1069969317 after retry RPC after error@223080375) -> (continuation of Deferred@725106148 after retry RPC after error@1907089839)))
21:03:41.700 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1088694356(state=PENDING, result=null, callback=(continuation of Deferred@2102977504 after retry RPC after error@1879419762) -> (continuation of Deferred@28051970 after retry RPC after error@1617638635) -> (continuation of Deferred@1519815909 after retry RPC after error@127263486) -> (continuation of Deferred@1720189972 after retry RPC after error@1610463298) -> (continuation of Deferred@836953852 after retry RPC after error@432562220) -> (continuation of Deferred@1069969317 after retry RPC after error@223080375) -> (continuation of Deferred@725106148 after retry RPC after error@1907089839), errback=(continuation of Deferred@2102977504 after retry RPC after error@1879419762) -> (continuation of Deferred@28051970 after retry RPC after error@1617638635) -> (continuation of Deferred@1519815909 after retry RPC after error@127263486) -> (continuation of Deferred@1720189972 after retry RPC after error@1610463298) -> (continuation of Deferred@836953852 after retry RPC after error@432562220) -> (continuation of Deferred@1069969317 after retry RPC after error@223080375) -> (continuation of Deferred@725106148 after retry RPC after error@1907089839)))
21:03:41.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.739 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.959 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.960 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@321865297(state=PENDING, result=null, callback=(continuation of Deferred@88517702 after retry RPC after error@1560972704) -> (continuation of Deferred@13539732 after retry RPC after error@1765431826) -> (continuation of Deferred@1496248303 after retry RPC after error@1354764007) -> (continuation of Deferred@607530834 after retry RPC after error@526846732) -> (continuation of Deferred@823713432 after retry RPC after error@1635731818) -> (continuation of Deferred@1957082792 after retry RPC after error@1396636763) -> (continuation of Deferred@614794426 after retry RPC after error@1057631581), errback=(continuation of Deferred@88517702 after retry RPC after error@1560972704) -> (continuation of Deferred@13539732 after retry RPC after error@1765431826) -> (continuation of Deferred@1496248303 after retry RPC after error@1354764007) -> (continuation of Deferred@607530834 after retry RPC after error@526846732) -> (continuation of Deferred@823713432 after retry RPC after error@1635731818) -> (continuation of Deferred@1957082792 after retry RPC after error@1396636763) -> (continuation of Deferred@614794426 after retry RPC after error@1057631581)))
21:03:41.961 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@321865297(state=PENDING, result=null, callback=(continuation of Deferred@88517702 after retry RPC after error@1560972704) -> (continuation of Deferred@13539732 after retry RPC after error@1765431826) -> (continuation of Deferred@1496248303 after retry RPC after error@1354764007) -> (continuation of Deferred@607530834 after retry RPC after error@526846732) -> (continuation of Deferred@823713432 after retry RPC after error@1635731818) -> (continuation of Deferred@1957082792 after retry RPC after error@1396636763) -> (continuation of Deferred@614794426 after retry RPC after error@1057631581), errback=(continuation of Deferred@88517702 after retry RPC after error@1560972704) -> (continuation of Deferred@13539732 after retry RPC after error@1765431826) -> (continuation of Deferred@1496248303 after retry RPC after error@1354764007) -> (continuation of Deferred@607530834 after retry RPC after error@526846732) -> (continuation of Deferred@823713432 after retry RPC after error@1635731818) -> (continuation of Deferred@1957082792 after retry RPC after error@1396636763) -> (continuation of Deferred@614794426 after retry RPC after error@1057631581)))
21:03:41.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.980 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:41.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:41.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:41.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:41.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:41.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.079 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.220 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1191593621(state=PENDING, result=null, callback=(continuation of Deferred@193445000 after retry RPC after error@204822257) -> (continuation of Deferred@2094024065 after retry RPC after error@160925280) -> (continuation of Deferred@1061292070 after retry RPC after error@434525153) -> (continuation of Deferred@1734664465 after retry RPC after error@1391841051) -> (continuation of Deferred@2111438427 after retry RPC after error@1452004186) -> (continuation of Deferred@380099827 after retry RPC after error@1792427051) -> (continuation of Deferred@1481593868 after retry RPC after error@1347833180) -> (continuation of Deferred@2017346529 after retry RPC after error@546416593), errback=(continuation of Deferred@193445000 after retry RPC after error@204822257) -> (continuation of Deferred@2094024065 after retry RPC after error@160925280) -> (continuation of Deferred@1061292070 after retry RPC after error@434525153) -> (continuation of Deferred@1734664465 after retry RPC after error@1391841051) -> (continuation of Deferred@2111438427 after retry RPC after error@1452004186) -> (continuation of Deferred@380099827 after retry RPC after error@1792427051) -> (continuation of Deferred@1481593868 after retry RPC after error@1347833180) -> (continuation of Deferred@2017346529 after retry RPC after error@546416593)))
21:03:42.220 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1191593621(state=PENDING, result=null, callback=(continuation of Deferred@193445000 after retry RPC after error@204822257) -> (continuation of Deferred@2094024065 after retry RPC after error@160925280) -> (continuation of Deferred@1061292070 after retry RPC after error@434525153) -> (continuation of Deferred@1734664465 after retry RPC after error@1391841051) -> (continuation of Deferred@2111438427 after retry RPC after error@1452004186) -> (continuation of Deferred@380099827 after retry RPC after error@1792427051) -> (continuation of Deferred@1481593868 after retry RPC after error@1347833180) -> (continuation of Deferred@2017346529 after retry RPC after error@546416593), errback=(continuation of Deferred@193445000 after retry RPC after error@204822257) -> (continuation of Deferred@2094024065 after retry RPC after error@160925280) -> (continuation of Deferred@1061292070 after retry RPC after error@434525153) -> (continuation of Deferred@1734664465 after retry RPC after error@1391841051) -> (continuation of Deferred@2111438427 after retry RPC after error@1452004186) -> (continuation of Deferred@380099827 after retry RPC after error@1792427051) -> (continuation of Deferred@1481593868 after retry RPC after error@1347833180) -> (continuation of Deferred@2017346529 after retry RPC after error@546416593)))
21:03:42.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.237 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.298 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.383 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.383 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.383 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.383 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.384 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.440 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [146ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [146ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [146ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [147ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2077677220(state=PENDING, result=null, callback=(continuation of Deferred@2040932806 after retry RPC after error@472491493) -> (continuation of Deferred@283380193 after retry RPC after error@1573555887) -> (continuation of Deferred@367682761 after retry RPC after error@1144909084) -> (continuation of Deferred@1948671867 after retry RPC after error@2098206886) -> (continuation of Deferred@1000007649 after retry RPC after error@283420803) -> (continuation of Deferred@1451221091 after retry RPC after error@735512354) -> (continuation of Deferred@1193171030 after retry RPC after error@1913344134), errback=(continuation of Deferred@2040932806 after retry RPC after error@472491493) -> (continuation of Deferred@283380193 after retry RPC after error@1573555887) -> (continuation of Deferred@367682761 after retry RPC after error@1144909084) -> (continuation of Deferred@1948671867 after retry RPC after error@2098206886) -> (continuation of Deferred@1000007649 after retry RPC after error@283420803) -> (continuation of Deferred@1451221091 after retry RPC after error@735512354) -> (continuation of Deferred@1193171030 after retry RPC after error@1913344134)))
21:03:42.440 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [146ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [146ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [146ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [147ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2077677220(state=PENDING, result=null, callback=(continuation of Deferred@2040932806 after retry RPC after error@472491493) -> (continuation of Deferred@283380193 after retry RPC after error@1573555887) -> (continuation of Deferred@367682761 after retry RPC after error@1144909084) -> (continuation of Deferred@1948671867 after retry RPC after error@2098206886) -> (continuation of Deferred@1000007649 after retry RPC after error@283420803) -> (continuation of Deferred@1451221091 after retry RPC after error@735512354) -> (continuation of Deferred@1193171030 after retry RPC after error@1913344134), errback=(continuation of Deferred@2040932806 after retry RPC after error@472491493) -> (continuation of Deferred@283380193 after retry RPC after error@1573555887) -> (continuation of Deferred@367682761 after retry RPC after error@1144909084) -> (continuation of Deferred@1948671867 after retry RPC after error@2098206886) -> (continuation of Deferred@1000007649 after retry RPC after error@283420803) -> (continuation of Deferred@1451221091 after retry RPC after error@735512354) -> (continuation of Deferred@1193171030 after retry RPC after error@1913344134)))
21:03:42.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.479 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.580 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:42.678953 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:42.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.679 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1233805724(state=PENDING, result=null, callback=(continuation of Deferred@1150019468 after retry RPC after error@975692907) -> (continuation of Deferred@1412823950 after retry RPC after error@1319757333) -> (continuation of Deferred@1426444235 after retry RPC after error@1350856124) -> (continuation of Deferred@722863529 after retry RPC after error@705615760) -> (continuation of Deferred@1368272525 after retry RPC after error@571752589) -> (continuation of Deferred@968110635 after retry RPC after error@1708835034) -> (continuation of Deferred@1157656746 after retry RPC after error@1167081647), errback=(continuation of Deferred@1150019468 after retry RPC after error@975692907) -> (continuation of Deferred@1412823950 after retry RPC after error@1319757333) -> (continuation of Deferred@1426444235 after retry RPC after error@1350856124) -> (continuation of Deferred@722863529 after retry RPC after error@705615760) -> (continuation of Deferred@1368272525 after retry RPC after error@571752589) -> (continuation of Deferred@968110635 after retry RPC after error@1708835034) -> (continuation of Deferred@1157656746 after retry RPC after error@1167081647)))
21:03:42.680 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1233805724(state=PENDING, result=null, callback=(continuation of Deferred@1150019468 after retry RPC after error@975692907) -> (continuation of Deferred@1412823950 after retry RPC after error@1319757333) -> (continuation of Deferred@1426444235 after retry RPC after error@1350856124) -> (continuation of Deferred@722863529 after retry RPC after error@705615760) -> (continuation of Deferred@1368272525 after retry RPC after error@571752589) -> (continuation of Deferred@968110635 after retry RPC after error@1708835034) -> (continuation of Deferred@1157656746 after retry RPC after error@1167081647), errback=(continuation of Deferred@1150019468 after retry RPC after error@975692907) -> (continuation of Deferred@1412823950 after retry RPC after error@1319757333) -> (continuation of Deferred@1426444235 after retry RPC after error@1350856124) -> (continuation of Deferred@722863529 after retry RPC after error@705615760) -> (continuation of Deferred@1368272525 after retry RPC after error@571752589) -> (continuation of Deferred@968110635 after retry RPC after error@1708835034) -> (continuation of Deferred@1157656746 after retry RPC after error@1167081647)))
21:03:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:42.684345 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:42.685671 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:03:42.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.738 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@980533400(state=PENDING, result=null, callback=(continuation of Deferred@1389634249 after retry RPC after error@2070689553) -> (continuation of Deferred@1301209463 after retry RPC after error@2043649374) -> (continuation of Deferred@176044740 after retry RPC after error@271601701) -> (continuation of Deferred@714055181 after retry RPC after error@800047983) -> (continuation of Deferred@1425826884 after retry RPC after error@1180673926) -> (continuation of Deferred@284567222 after retry RPC after error@1100168060), errback=(continuation of Deferred@1389634249 after retry RPC after error@2070689553) -> (continuation of Deferred@1301209463 after retry RPC after error@2043649374) -> (continuation of Deferred@176044740 after retry RPC after error@271601701) -> (continuation of Deferred@714055181 after retry RPC after error@800047983) -> (continuation of Deferred@1425826884 after retry RPC after error@1180673926) -> (continuation of Deferred@284567222 after retry RPC after error@1100168060)))
21:03:42.880 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@980533400(state=PENDING, result=null, callback=(continuation of Deferred@1389634249 after retry RPC after error@2070689553) -> (continuation of Deferred@1301209463 after retry RPC after error@2043649374) -> (continuation of Deferred@176044740 after retry RPC after error@271601701) -> (continuation of Deferred@714055181 after retry RPC after error@800047983) -> (continuation of Deferred@1425826884 after retry RPC after error@1180673926) -> (continuation of Deferred@284567222 after retry RPC after error@1100168060), errback=(continuation of Deferred@1389634249 after retry RPC after error@2070689553) -> (continuation of Deferred@1301209463 after retry RPC after error@2043649374) -> (continuation of Deferred@176044740 after retry RPC after error@271601701) -> (continuation of Deferred@714055181 after retry RPC after error@800047983) -> (continuation of Deferred@1425826884 after retry RPC after error@1180673926) -> (continuation of Deferred@284567222 after retry RPC after error@1100168060)))
21:03:42.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:42.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:42.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:42.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:42.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:42.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.040 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@690269110(state=PENDING, result=null, callback=(continuation of Deferred@255522041 after retry RPC after error@1176951861) -> (continuation of Deferred@595262315 after retry RPC after error@797941161) -> (continuation of Deferred@2001349021 after retry RPC after error@125666357) -> (continuation of Deferred@1932035602 after retry RPC after error@546909963) -> (continuation of Deferred@104459332 after retry RPC after error@1838058650) -> (continuation of Deferred@1512148783 after retry RPC after error@858325527), errback=(continuation of Deferred@255522041 after retry RPC after error@1176951861) -> (continuation of Deferred@595262315 after retry RPC after error@797941161) -> (continuation of Deferred@2001349021 after retry RPC after error@125666357) -> (continuation of Deferred@1932035602 after retry RPC after error@546909963) -> (continuation of Deferred@104459332 after retry RPC after error@1838058650) -> (continuation of Deferred@1512148783 after retry RPC after error@858325527)))
21:03:43.041 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@690269110(state=PENDING, result=null, callback=(continuation of Deferred@255522041 after retry RPC after error@1176951861) -> (continuation of Deferred@595262315 after retry RPC after error@797941161) -> (continuation of Deferred@2001349021 after retry RPC after error@125666357) -> (continuation of Deferred@1932035602 after retry RPC after error@546909963) -> (continuation of Deferred@104459332 after retry RPC after error@1838058650) -> (continuation of Deferred@1512148783 after retry RPC after error@858325527), errback=(continuation of Deferred@255522041 after retry RPC after error@1176951861) -> (continuation of Deferred@595262315 after retry RPC after error@797941161) -> (continuation of Deferred@2001349021 after retry RPC after error@125666357) -> (continuation of Deferred@1932035602 after retry RPC after error@546909963) -> (continuation of Deferred@104459332 after retry RPC after error@1838058650) -> (continuation of Deferred@1512148783 after retry RPC after error@858325527)))
21:03:43.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.059 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.060 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.260 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@643540540(state=PENDING, result=null, callback=(continuation of Deferred@682596228 after retry RPC after error@1446535564) -> (continuation of Deferred@466005245 after retry RPC after error@1616352255) -> (continuation of Deferred@873479110 after retry RPC after error@556861641) -> (continuation of Deferred@255781206 after retry RPC after error@1395284773) -> (continuation of Deferred@1716622477 after retry RPC after error@1334820814) -> (continuation of Deferred@931371824 after retry RPC after error@913382806) -> (continuation of Deferred@1559106089 after retry RPC after error@1856874421), errback=(continuation of Deferred@682596228 after retry RPC after error@1446535564) -> (continuation of Deferred@466005245 after retry RPC after error@1616352255) -> (continuation of Deferred@873479110 after retry RPC after error@556861641) -> (continuation of Deferred@255781206 after retry RPC after error@1395284773) -> (continuation of Deferred@1716622477 after retry RPC after error@1334820814) -> (continuation of Deferred@931371824 after retry RPC after error@913382806) -> (continuation of Deferred@1559106089 after retry RPC after error@1856874421)))
21:03:43.260 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@643540540(state=PENDING, result=null, callback=(continuation of Deferred@682596228 after retry RPC after error@1446535564) -> (continuation of Deferred@466005245 after retry RPC after error@1616352255) -> (continuation of Deferred@873479110 after retry RPC after error@556861641) -> (continuation of Deferred@255781206 after retry RPC after error@1395284773) -> (continuation of Deferred@1716622477 after retry RPC after error@1334820814) -> (continuation of Deferred@931371824 after retry RPC after error@913382806) -> (continuation of Deferred@1559106089 after retry RPC after error@1856874421), errback=(continuation of Deferred@682596228 after retry RPC after error@1446535564) -> (continuation of Deferred@466005245 after retry RPC after error@1616352255) -> (continuation of Deferred@873479110 after retry RPC after error@556861641) -> (continuation of Deferred@255781206 after retry RPC after error@1395284773) -> (continuation of Deferred@1716622477 after retry RPC after error@1334820814) -> (continuation of Deferred@931371824 after retry RPC after error@913382806) -> (continuation of Deferred@1559106089 after retry RPC after error@1856874421)))
21:03:43.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.277 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.537 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1115497327(state=PENDING, result=null, callback=(continuation of Deferred@721232553 after retry RPC after error@1617823631) -> (continuation of Deferred@900617977 after retry RPC after error@518225272) -> (continuation of Deferred@1565060199 after retry RPC after error@1300945470) -> (continuation of Deferred@173298920 after retry RPC after error@265941095) -> (continuation of Deferred@359140911 after retry RPC after error@237530518) -> (continuation of Deferred@653499311 after retry RPC after error@1718164958) -> (continuation of Deferred@866483786 after retry RPC after error@2116874613), errback=(continuation of Deferred@721232553 after retry RPC after error@1617823631) -> (continuation of Deferred@900617977 after retry RPC after error@518225272) -> (continuation of Deferred@1565060199 after retry RPC after error@1300945470) -> (continuation of Deferred@173298920 after retry RPC after error@265941095) -> (continuation of Deferred@359140911 after retry RPC after error@237530518) -> (continuation of Deferred@653499311 after retry RPC after error@1718164958) -> (continuation of Deferred@866483786 after retry RPC after error@2116874613)))
21:03:43.538 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:43.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.637 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.638 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.699 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1633051287(state=PENDING, result=null, callback=(continuation of Deferred@2141319171 after retry RPC after error@2025059392) -> (continuation of Deferred@1523382003 after retry RPC after error@148665831) -> (continuation of Deferred@2014162199 after retry RPC after error@528172962) -> (continuation of Deferred@582832592 after retry RPC after error@1235581741) -> (continuation of Deferred@1557467675 after retry RPC after error@425072756) -> (continuation of Deferred@1361967455 after retry RPC after error@867408964), errback=(continuation of Deferred@2141319171 after retry RPC after error@2025059392) -> (continuation of Deferred@1523382003 after retry RPC after error@148665831) -> (continuation of Deferred@2014162199 after retry RPC after error@528172962) -> (continuation of Deferred@582832592 after retry RPC after error@1235581741) -> (continuation of Deferred@1557467675 after retry RPC after error@425072756) -> (continuation of Deferred@1361967455 after retry RPC after error@867408964)))
21:03:43.700 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1633051287(state=PENDING, result=null, callback=(continuation of Deferred@2141319171 after retry RPC after error@2025059392) -> (continuation of Deferred@1523382003 after retry RPC after error@148665831) -> (continuation of Deferred@2014162199 after retry RPC after error@528172962) -> (continuation of Deferred@582832592 after retry RPC after error@1235581741) -> (continuation of Deferred@1557467675 after retry RPC after error@425072756) -> (continuation of Deferred@1361967455 after retry RPC after error@867408964), errback=(continuation of Deferred@2141319171 after retry RPC after error@2025059392) -> (continuation of Deferred@1523382003 after retry RPC after error@148665831) -> (continuation of Deferred@2014162199 after retry RPC after error@528172962) -> (continuation of Deferred@582832592 after retry RPC after error@1235581741) -> (continuation of Deferred@1557467675 after retry RPC after error@425072756) -> (continuation of Deferred@1361967455 after retry RPC after error@867408964)))
21:03:43.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.940 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@156832514(state=PENDING, result=null, callback=(continuation of Deferred@1910437243 after retry RPC after error@1574060884) -> (continuation of Deferred@590779572 after retry RPC after error@1656115923) -> (continuation of Deferred@1690633939 after retry RPC after error@734142972) -> (continuation of Deferred@1019646724 after retry RPC after error@1815510661) -> (continuation of Deferred@545853006 after retry RPC after error@452859490) -> (continuation of Deferred@313063562 after retry RPC after error@1295347212) -> (continuation of Deferred@623305370 after retry RPC after error@1333902682), errback=(continuation of Deferred@1910437243 after retry RPC after error@1574060884) -> (continuation of Deferred@590779572 after retry RPC after error@1656115923) -> (continuation of Deferred@1690633939 after retry RPC after error@734142972) -> (continuation of Deferred@1019646724 after retry RPC after error@1815510661) -> (continuation of Deferred@545853006 after retry RPC after error@452859490) -> (continuation of Deferred@313063562 after retry RPC after error@1295347212) -> (continuation of Deferred@623305370 after retry RPC after error@1333902682)))
21:03:43.940 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@156832514(state=PENDING, result=null, callback=(continuation of Deferred@1910437243 after retry RPC after error@1574060884) -> (continuation of Deferred@590779572 after retry RPC after error@1656115923) -> (continuation of Deferred@1690633939 after retry RPC after error@734142972) -> (continuation of Deferred@1019646724 after retry RPC after error@1815510661) -> (continuation of Deferred@545853006 after retry RPC after error@452859490) -> (continuation of Deferred@313063562 after retry RPC after error@1295347212) -> (continuation of Deferred@623305370 after retry RPC after error@1333902682), errback=(continuation of Deferred@1910437243 after retry RPC after error@1574060884) -> (continuation of Deferred@590779572 after retry RPC after error@1656115923) -> (continuation of Deferred@1690633939 after retry RPC after error@734142972) -> (continuation of Deferred@1019646724 after retry RPC after error@1815510661) -> (continuation of Deferred@545853006 after retry RPC after error@452859490) -> (continuation of Deferred@313063562 after retry RPC after error@1295347212) -> (continuation of Deferred@623305370 after retry RPC after error@1333902682)))
21:03:43.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.959 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:43.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:43.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:43.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:43.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:43.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.180 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1361192943(state=PENDING, result=null, callback=(continuation of Deferred@834827256 after retry RPC after error@1724825406) -> (continuation of Deferred@535268099 after retry RPC after error@1606081103) -> (continuation of Deferred@2101686259 after retry RPC after error@1860261930) -> (continuation of Deferred@1308453222 after retry RPC after error@1151117876) -> (continuation of Deferred@284307427 after retry RPC after error@1949697503) -> (continuation of Deferred@1378284646 after retry RPC after error@1934244124) -> (continuation of Deferred@1917230497 after retry RPC after error@907704011), errback=(continuation of Deferred@834827256 after retry RPC after error@1724825406) -> (continuation of Deferred@535268099 after retry RPC after error@1606081103) -> (continuation of Deferred@2101686259 after retry RPC after error@1860261930) -> (continuation of Deferred@1308453222 after retry RPC after error@1151117876) -> (continuation of Deferred@284307427 after retry RPC after error@1949697503) -> (continuation of Deferred@1378284646 after retry RPC after error@1934244124) -> (continuation of Deferred@1917230497 after retry RPC after error@907704011)))
21:03:44.180 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1361192943(state=PENDING, result=null, callback=(continuation of Deferred@834827256 after retry RPC after error@1724825406) -> (continuation of Deferred@535268099 after retry RPC after error@1606081103) -> (continuation of Deferred@2101686259 after retry RPC after error@1860261930) -> (continuation of Deferred@1308453222 after retry RPC after error@1151117876) -> (continuation of Deferred@284307427 after retry RPC after error@1949697503) -> (continuation of Deferred@1378284646 after retry RPC after error@1934244124) -> (continuation of Deferred@1917230497 after retry RPC after error@907704011), errback=(continuation of Deferred@834827256 after retry RPC after error@1724825406) -> (continuation of Deferred@535268099 after retry RPC after error@1606081103) -> (continuation of Deferred@2101686259 after retry RPC after error@1860261930) -> (continuation of Deferred@1308453222 after retry RPC after error@1151117876) -> (continuation of Deferred@284307427 after retry RPC after error@1949697503) -> (continuation of Deferred@1378284646 after retry RPC after error@1934244124) -> (continuation of Deferred@1917230497 after retry RPC after error@907704011)))
21:03:44.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.220 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.220 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.220 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.440 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@82986744(state=PENDING, result=null, callback=(continuation of Deferred@1595392680 after retry RPC after error@1946735538) -> (continuation of Deferred@2146010938 after retry RPC after error@1773853625) -> (continuation of Deferred@148461220 after retry RPC after error@989757774) -> (continuation of Deferred@161288572 after retry RPC after error@910517477) -> (continuation of Deferred@1961584895 after retry RPC after error@978412509) -> (continuation of Deferred@1537639194 after retry RPC after error@1180284615) -> (continuation of Deferred@1775927907 after retry RPC after error@1478773518), errback=(continuation of Deferred@1595392680 after retry RPC after error@1946735538) -> (continuation of Deferred@2146010938 after retry RPC after error@1773853625) -> (continuation of Deferred@148461220 after retry RPC after error@989757774) -> (continuation of Deferred@161288572 after retry RPC after error@910517477) -> (continuation of Deferred@1961584895 after retry RPC after error@978412509) -> (continuation of Deferred@1537639194 after retry RPC after error@1180284615) -> (continuation of Deferred@1775927907 after retry RPC after error@1478773518)))
21:03:44.440 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@82986744(state=PENDING, result=null, callback=(continuation of Deferred@1595392680 after retry RPC after error@1946735538) -> (continuation of Deferred@2146010938 after retry RPC after error@1773853625) -> (continuation of Deferred@148461220 after retry RPC after error@989757774) -> (continuation of Deferred@161288572 after retry RPC after error@910517477) -> (continuation of Deferred@1961584895 after retry RPC after error@978412509) -> (continuation of Deferred@1537639194 after retry RPC after error@1180284615) -> (continuation of Deferred@1775927907 after retry RPC after error@1478773518), errback=(continuation of Deferred@1595392680 after retry RPC after error@1946735538) -> (continuation of Deferred@2146010938 after retry RPC after error@1773853625) -> (continuation of Deferred@148461220 after retry RPC after error@989757774) -> (continuation of Deferred@161288572 after retry RPC after error@910517477) -> (continuation of Deferred@1961584895 after retry RPC after error@978412509) -> (continuation of Deferred@1537639194 after retry RPC after error@1180284615) -> (continuation of Deferred@1775927907 after retry RPC after error@1478773518)))
21:03:44.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.699 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2078682684(state=PENDING, result=null, callback=(continuation of Deferred@637168593 after retry RPC after error@2027857520) -> (continuation of Deferred@1776325518 after retry RPC after error@591251264) -> (continuation of Deferred@1441760277 after retry RPC after error@1477395966) -> (continuation of Deferred@891212178 after retry RPC after error@2027165622) -> (continuation of Deferred@1362291605 after retry RPC after error@1932551416) -> (continuation of Deferred@811706701 after retry RPC after error@1750098256) -> (continuation of Deferred@779633036 after retry RPC after error@1579153183), errback=(continuation of Deferred@637168593 after retry RPC after error@2027857520) -> (continuation of Deferred@1776325518 after retry RPC after error@591251264) -> (continuation of Deferred@1441760277 after retry RPC after error@1477395966) -> (continuation of Deferred@891212178 after retry RPC after error@2027165622) -> (continuation of Deferred@1362291605 after retry RPC after error@1932551416) -> (continuation of Deferred@811706701 after retry RPC after error@1750098256) -> (continuation of Deferred@779633036 after retry RPC after error@1579153183)))
21:03:44.700 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2078682684(state=PENDING, result=null, callback=(continuation of Deferred@637168593 after retry RPC after error@2027857520) -> (continuation of Deferred@1776325518 after retry RPC after error@591251264) -> (continuation of Deferred@1441760277 after retry RPC after error@1477395966) -> (continuation of Deferred@891212178 after retry RPC after error@2027165622) -> (continuation of Deferred@1362291605 after retry RPC after error@1932551416) -> (continuation of Deferred@811706701 after retry RPC after error@1750098256) -> (continuation of Deferred@779633036 after retry RPC after error@1579153183), errback=(continuation of Deferred@637168593 after retry RPC after error@2027857520) -> (continuation of Deferred@1776325518 after retry RPC after error@591251264) -> (continuation of Deferred@1441760277 after retry RPC after error@1477395966) -> (continuation of Deferred@891212178 after retry RPC after error@2027165622) -> (continuation of Deferred@1362291605 after retry RPC after error@1932551416) -> (continuation of Deferred@811706701 after retry RPC after error@1750098256) -> (continuation of Deferred@779633036 after retry RPC after error@1579153183)))
21:03:44.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.739 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@569797609(state=PENDING, result=null, callback=(continuation of Deferred@2085902138 after retry RPC after error@1532750467) -> (continuation of Deferred@443300271 after retry RPC after error@148353064) -> (continuation of Deferred@1449450383 after retry RPC after error@1383017522) -> (continuation of Deferred@699778705 after retry RPC after error@475424127) -> (continuation of Deferred@1822322137 after retry RPC after error@1203487428) -> (continuation of Deferred@1086133740 after retry RPC after error@1877998873), errback=(continuation of Deferred@2085902138 after retry RPC after error@1532750467) -> (continuation of Deferred@443300271 after retry RPC after error@148353064) -> (continuation of Deferred@1449450383 after retry RPC after error@1383017522) -> (continuation of Deferred@699778705 after retry RPC after error@475424127) -> (continuation of Deferred@1822322137 after retry RPC after error@1203487428) -> (continuation of Deferred@1086133740 after retry RPC after error@1877998873)))
21:03:44.881 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@569797609(state=PENDING, result=null, callback=(continuation of Deferred@2085902138 after retry RPC after error@1532750467) -> (continuation of Deferred@443300271 after retry RPC after error@148353064) -> (continuation of Deferred@1449450383 after retry RPC after error@1383017522) -> (continuation of Deferred@699778705 after retry RPC after error@475424127) -> (continuation of Deferred@1822322137 after retry RPC after error@1203487428) -> (continuation of Deferred@1086133740 after retry RPC after error@1877998873), errback=(continuation of Deferred@2085902138 after retry RPC after error@1532750467) -> (continuation of Deferred@443300271 after retry RPC after error@148353064) -> (continuation of Deferred@1449450383 after retry RPC after error@1383017522) -> (continuation of Deferred@699778705 after retry RPC after error@475424127) -> (continuation of Deferred@1822322137 after retry RPC after error@1203487428) -> (continuation of Deferred@1086133740 after retry RPC after error@1877998873)))
21:03:44.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:44.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:44.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:44.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:44.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:44.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.099 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1395251216(state=PENDING, result=null, callback=(continuation of Deferred@1098214129 after retry RPC after error@879963480) -> (continuation of Deferred@154356849 after retry RPC after error@340261919) -> (continuation of Deferred@2123312772 after retry RPC after error@1404818912) -> (continuation of Deferred@1081625378 after retry RPC after error@1802825782) -> (continuation of Deferred@1820854482 after retry RPC after error@1628554358) -> (continuation of Deferred@1664262701 after retry RPC after error@789252925) -> (continuation of Deferred@65624214 after retry RPC after error@1823743721), errback=(continuation of Deferred@1098214129 after retry RPC after error@879963480) -> (continuation of Deferred@154356849 after retry RPC after error@340261919) -> (continuation of Deferred@2123312772 after retry RPC after error@1404818912) -> (continuation of Deferred@1081625378 after retry RPC after error@1802825782) -> (continuation of Deferred@1820854482 after retry RPC after error@1628554358) -> (continuation of Deferred@1664262701 after retry RPC after error@789252925) -> (continuation of Deferred@65624214 after retry RPC after error@1823743721)))
21:03:45.099 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1395251216(state=PENDING, result=null, callback=(continuation of Deferred@1098214129 after retry RPC after error@879963480) -> (continuation of Deferred@154356849 after retry RPC after error@340261919) -> (continuation of Deferred@2123312772 after retry RPC after error@1404818912) -> (continuation of Deferred@1081625378 after retry RPC after error@1802825782) -> (continuation of Deferred@1820854482 after retry RPC after error@1628554358) -> (continuation of Deferred@1664262701 after retry RPC after error@789252925) -> (continuation of Deferred@65624214 after retry RPC after error@1823743721), errback=(continuation of Deferred@1098214129 after retry RPC after error@879963480) -> (continuation of Deferred@154356849 after retry RPC after error@340261919) -> (continuation of Deferred@2123312772 after retry RPC after error@1404818912) -> (continuation of Deferred@1081625378 after retry RPC after error@1802825782) -> (continuation of Deferred@1820854482 after retry RPC after error@1628554358) -> (continuation of Deferred@1664262701 after retry RPC after error@789252925) -> (continuation of Deferred@65624214 after retry RPC after error@1823743721)))
21:03:45.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.179 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.179 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.179 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.340 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.340 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1764416530(state=PENDING, result=null, callback=(continuation of Deferred@619849147 after retry RPC after error@682008312) -> (continuation of Deferred@300142557 after retry RPC after error@773213119) -> (continuation of Deferred@469527611 after retry RPC after error@120676578) -> (continuation of Deferred@144952062 after retry RPC after error@343155686) -> (continuation of Deferred@1813055719 after retry RPC after error@1613452279) -> (continuation of Deferred@1984528694 after retry RPC after error@1134032650) -> (continuation of Deferred@665160111 after retry RPC after error@497795655), errback=(continuation of Deferred@619849147 after retry RPC after error@682008312) -> (continuation of Deferred@300142557 after retry RPC after error@773213119) -> (continuation of Deferred@469527611 after retry RPC after error@120676578) -> (continuation of Deferred@144952062 after retry RPC after error@343155686) -> (continuation of Deferred@1813055719 after retry RPC after error@1613452279) -> (continuation of Deferred@1984528694 after retry RPC after error@1134032650) -> (continuation of Deferred@665160111 after retry RPC after error@497795655)))
21:03:45.341 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1764416530(state=PENDING, result=null, callback=(continuation of Deferred@619849147 after retry RPC after error@682008312) -> (continuation of Deferred@300142557 after retry RPC after error@773213119) -> (continuation of Deferred@469527611 after retry RPC after error@120676578) -> (continuation of Deferred@144952062 after retry RPC after error@343155686) -> (continuation of Deferred@1813055719 after retry RPC after error@1613452279) -> (continuation of Deferred@1984528694 after retry RPC after error@1134032650) -> (continuation of Deferred@665160111 after retry RPC after error@497795655), errback=(continuation of Deferred@619849147 after retry RPC after error@682008312) -> (continuation of Deferred@300142557 after retry RPC after error@773213119) -> (continuation of Deferred@469527611 after retry RPC after error@120676578) -> (continuation of Deferred@144952062 after retry RPC after error@343155686) -> (continuation of Deferred@1813055719 after retry RPC after error@1613452279) -> (continuation of Deferred@1984528694 after retry RPC after error@1134032650) -> (continuation of Deferred@665160111 after retry RPC after error@497795655)))
21:03:45.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.397 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.398 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.580 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1669904426(state=PENDING, result=null, callback=(continuation of Deferred@1996867043 after retry RPC after error@635093289) -> (continuation of Deferred@1352311190 after retry RPC after error@1987381831) -> (continuation of Deferred@788980663 after retry RPC after error@1554427127) -> (continuation of Deferred@1505046756 after retry RPC after error@130821799) -> (continuation of Deferred@525299378 after retry RPC after error@1986504888) -> (continuation of Deferred@111510656 after retry RPC after error@1618623461) -> (continuation of Deferred@77915046 after retry RPC after error@1338878434), errback=(continuation of Deferred@1996867043 after retry RPC after error@635093289) -> (continuation of Deferred@1352311190 after retry RPC after error@1987381831) -> (continuation of Deferred@788980663 after retry RPC after error@1554427127) -> (continuation of Deferred@1505046756 after retry RPC after error@130821799) -> (continuation of Deferred@525299378 after retry RPC after error@1986504888) -> (continuation of Deferred@111510656 after retry RPC after error@1618623461) -> (continuation of Deferred@77915046 after retry RPC after error@1338878434)))
21:03:45.580 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1669904426(state=PENDING, result=null, callback=(continuation of Deferred@1996867043 after retry RPC after error@635093289) -> (continuation of Deferred@1352311190 after retry RPC after error@1987381831) -> (continuation of Deferred@788980663 after retry RPC after error@1554427127) -> (continuation of Deferred@1505046756 after retry RPC after error@130821799) -> (continuation of Deferred@525299378 after retry RPC after error@1986504888) -> (continuation of Deferred@111510656 after retry RPC after error@1618623461) -> (continuation of Deferred@77915046 after retry RPC after error@1338878434), errback=(continuation of Deferred@1996867043 after retry RPC after error@635093289) -> (continuation of Deferred@1352311190 after retry RPC after error@1987381831) -> (continuation of Deferred@788980663 after retry RPC after error@1554427127) -> (continuation of Deferred@1505046756 after retry RPC after error@130821799) -> (continuation of Deferred@525299378 after retry RPC after error@1986504888) -> (continuation of Deferred@111510656 after retry RPC after error@1618623461) -> (continuation of Deferred@77915046 after retry RPC after error@1338878434)))
21:03:45.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.640 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.760 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1530190585(state=PENDING, result=null, callback=(continuation of Deferred@2089901199 after retry RPC after error@1635734249) -> (continuation of Deferred@713846797 after retry RPC after error@1460835511) -> (continuation of Deferred@563208190 after retry RPC after error@987900223) -> (continuation of Deferred@1931946614 after retry RPC after error@819183497) -> (continuation of Deferred@521336898 after retry RPC after error@873632078) -> (continuation of Deferred@2116843808 after retry RPC after error@1403956891) -> (continuation of Deferred@1534495812 after retry RPC after error@2068770437), errback=(continuation of Deferred@2089901199 after retry RPC after error@1635734249) -> (continuation of Deferred@713846797 after retry RPC after error@1460835511) -> (continuation of Deferred@563208190 after retry RPC after error@987900223) -> (continuation of Deferred@1931946614 after retry RPC after error@819183497) -> (continuation of Deferred@521336898 after retry RPC after error@873632078) -> (continuation of Deferred@2116843808 after retry RPC after error@1403956891) -> (continuation of Deferred@1534495812 after retry RPC after error@2068770437)))
21:03:45.761 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1530190585(state=PENDING, result=null, callback=(continuation of Deferred@2089901199 after retry RPC after error@1635734249) -> (continuation of Deferred@713846797 after retry RPC after error@1460835511) -> (continuation of Deferred@563208190 after retry RPC after error@987900223) -> (continuation of Deferred@1931946614 after retry RPC after error@819183497) -> (continuation of Deferred@521336898 after retry RPC after error@873632078) -> (continuation of Deferred@2116843808 after retry RPC after error@1403956891) -> (continuation of Deferred@1534495812 after retry RPC after error@2068770437), errback=(continuation of Deferred@2089901199 after retry RPC after error@1635734249) -> (continuation of Deferred@713846797 after retry RPC after error@1460835511) -> (continuation of Deferred@563208190 after retry RPC after error@987900223) -> (continuation of Deferred@1931946614 after retry RPC after error@819183497) -> (continuation of Deferred@521336898 after retry RPC after error@873632078) -> (continuation of Deferred@2116843808 after retry RPC after error@1403956891) -> (continuation of Deferred@1534495812 after retry RPC after error@2068770437)))
21:03:45.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.979 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:45.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:45.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:45.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:45.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:45.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.000 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1117662395(state=PENDING, result=null, callback=(continuation of Deferred@1637051589 after retry RPC after error@1112316519) -> (continuation of Deferred@658908927 after retry RPC after error@336473432) -> (continuation of Deferred@1845595736 after retry RPC after error@1211678301) -> (continuation of Deferred@1583286395 after retry RPC after error@1004444165) -> (continuation of Deferred@1506779240 after retry RPC after error@1409824602) -> (continuation of Deferred@1915941232 after retry RPC after error@1039352705) -> (continuation of Deferred@421230775 after retry RPC after error@213663273) -> (continuation of Deferred@1776347071 after retry RPC after error@1311264329), errback=(continuation of Deferred@1637051589 after retry RPC after error@1112316519) -> (continuation of Deferred@658908927 after retry RPC after error@336473432) -> (continuation of Deferred@1845595736 after retry RPC after error@1211678301) -> (continuation of Deferred@1583286395 after retry RPC after error@1004444165) -> (continuation of Deferred@1506779240 after retry RPC after error@1409824602) -> (continuation of Deferred@1915941232 after retry RPC after error@1039352705) -> (continuation of Deferred@421230775 after retry RPC after error@213663273) -> (continuation of Deferred@1776347071 after retry RPC after error@1311264329)))
21:03:46.000 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1117662395(state=PENDING, result=null, callback=(continuation of Deferred@1637051589 after retry RPC after error@1112316519) -> (continuation of Deferred@658908927 after retry RPC after error@336473432) -> (continuation of Deferred@1845595736 after retry RPC after error@1211678301) -> (continuation of Deferred@1583286395 after retry RPC after error@1004444165) -> (continuation of Deferred@1506779240 after retry RPC after error@1409824602) -> (continuation of Deferred@1915941232 after retry RPC after error@1039352705) -> (continuation of Deferred@421230775 after retry RPC after error@213663273) -> (continuation of Deferred@1776347071 after retry RPC after error@1311264329), errback=(continuation of Deferred@1637051589 after retry RPC after error@1112316519) -> (continuation of Deferred@658908927 after retry RPC after error@336473432) -> (continuation of Deferred@1845595736 after retry RPC after error@1211678301) -> (continuation of Deferred@1583286395 after retry RPC after error@1004444165) -> (continuation of Deferred@1506779240 after retry RPC after error@1409824602) -> (continuation of Deferred@1915941232 after retry RPC after error@1039352705) -> (continuation of Deferred@421230775 after retry RPC after error@213663273) -> (continuation of Deferred@1776347071 after retry RPC after error@1311264329)))
21:03:46.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.098 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.199 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.237 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.239 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1185844672(state=PENDING, result=null, callback=(continuation of Deferred@525124225 after retry RPC after error@1385490992) -> (continuation of Deferred@1973901982 after retry RPC after error@1196759399) -> (continuation of Deferred@1047823302 after retry RPC after error@1262758738) -> (continuation of Deferred@75033358 after retry RPC after error@1795769543) -> (continuation of Deferred@1912180947 after retry RPC after error@453847813) -> (continuation of Deferred@1483641841 after retry RPC after error@2036386631) -> (continuation of Deferred@1143124912 after retry RPC after error@906358066), errback=(continuation of Deferred@525124225 after retry RPC after error@1385490992) -> (continuation of Deferred@1973901982 after retry RPC after error@1196759399) -> (continuation of Deferred@1047823302 after retry RPC after error@1262758738) -> (continuation of Deferred@75033358 after retry RPC after error@1795769543) -> (continuation of Deferred@1912180947 after retry RPC after error@453847813) -> (continuation of Deferred@1483641841 after retry RPC after error@2036386631) -> (continuation of Deferred@1143124912 after retry RPC after error@906358066)))
21:03:46.240 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1185844672(state=PENDING, result=null, callback=(continuation of Deferred@525124225 after retry RPC after error@1385490992) -> (continuation of Deferred@1973901982 after retry RPC after error@1196759399) -> (continuation of Deferred@1047823302 after retry RPC after error@1262758738) -> (continuation of Deferred@75033358 after retry RPC after error@1795769543) -> (continuation of Deferred@1912180947 after retry RPC after error@453847813) -> (continuation of Deferred@1483641841 after retry RPC after error@2036386631) -> (continuation of Deferred@1143124912 after retry RPC after error@906358066), errback=(continuation of Deferred@525124225 after retry RPC after error@1385490992) -> (continuation of Deferred@1973901982 after retry RPC after error@1196759399) -> (continuation of Deferred@1047823302 after retry RPC after error@1262758738) -> (continuation of Deferred@75033358 after retry RPC after error@1795769543) -> (continuation of Deferred@1912180947 after retry RPC after error@453847813) -> (continuation of Deferred@1483641841 after retry RPC after error@2036386631) -> (continuation of Deferred@1143124912 after retry RPC after error@906358066)))
21:03:46.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.259 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.338 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.499 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1040982065(state=PENDING, result=null, callback=(continuation of Deferred@2005815658 after retry RPC after error@152060729) -> (continuation of Deferred@916079409 after retry RPC after error@1774776443) -> (continuation of Deferred@1816717609 after retry RPC after error@1741317650) -> (continuation of Deferred@53293585 after retry RPC after error@768098599) -> (continuation of Deferred@115532822 after retry RPC after error@850192430) -> (continuation of Deferred@1492908496 after retry RPC after error@928336529) -> (continuation of Deferred@646371897 after retry RPC after error@1941875551), errback=(continuation of Deferred@2005815658 after retry RPC after error@152060729) -> (continuation of Deferred@916079409 after retry RPC after error@1774776443) -> (continuation of Deferred@1816717609 after retry RPC after error@1741317650) -> (continuation of Deferred@53293585 after retry RPC after error@768098599) -> (continuation of Deferred@115532822 after retry RPC after error@850192430) -> (continuation of Deferred@1492908496 after retry RPC after error@928336529) -> (continuation of Deferred@646371897 after retry RPC after error@1941875551)))
21:03:46.500 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1040982065(state=PENDING, result=null, callback=(continuation of Deferred@2005815658 after retry RPC after error@152060729) -> (continuation of Deferred@916079409 after retry RPC after error@1774776443) -> (continuation of Deferred@1816717609 after retry RPC after error@1741317650) -> (continuation of Deferred@53293585 after retry RPC after error@768098599) -> (continuation of Deferred@115532822 after retry RPC after error@850192430) -> (continuation of Deferred@1492908496 after retry RPC after error@928336529) -> (continuation of Deferred@646371897 after retry RPC after error@1941875551), errback=(continuation of Deferred@2005815658 after retry RPC after error@152060729) -> (continuation of Deferred@916079409 after retry RPC after error@1774776443) -> (continuation of Deferred@1816717609 after retry RPC after error@1741317650) -> (continuation of Deferred@53293585 after retry RPC after error@768098599) -> (continuation of Deferred@115532822 after retry RPC after error@850192430) -> (continuation of Deferred@1492908496 after retry RPC after error@928336529) -> (continuation of Deferred@646371897 after retry RPC after error@1941875551)))
21:03:46.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.517 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.518 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.740 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1725183234(state=PENDING, result=null, callback=(continuation of Deferred@1420077921 after retry RPC after error@1793247907) -> (continuation of Deferred@2029455100 after retry RPC after error@1061966021) -> (continuation of Deferred@1312955374 after retry RPC after error@839481175) -> (continuation of Deferred@1913210636 after retry RPC after error@790559728) -> (continuation of Deferred@1910315158 after retry RPC after error@144144949) -> (continuation of Deferred@815688972 after retry RPC after error@1745342868) -> (continuation of Deferred@1896312234 after retry RPC after error@1123219309), errback=(continuation of Deferred@1420077921 after retry RPC after error@1793247907) -> (continuation of Deferred@2029455100 after retry RPC after error@1061966021) -> (continuation of Deferred@1312955374 after retry RPC after error@839481175) -> (continuation of Deferred@1913210636 after retry RPC after error@790559728) -> (continuation of Deferred@1910315158 after retry RPC after error@144144949) -> (continuation of Deferred@815688972 after retry RPC after error@1745342868) -> (continuation of Deferred@1896312234 after retry RPC after error@1123219309)))
21:03:46.741 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1725183234(state=PENDING, result=null, callback=(continuation of Deferred@1420077921 after retry RPC after error@1793247907) -> (continuation of Deferred@2029455100 after retry RPC after error@1061966021) -> (continuation of Deferred@1312955374 after retry RPC after error@839481175) -> (continuation of Deferred@1913210636 after retry RPC after error@790559728) -> (continuation of Deferred@1910315158 after retry RPC after error@144144949) -> (continuation of Deferred@815688972 after retry RPC after error@1745342868) -> (continuation of Deferred@1896312234 after retry RPC after error@1123219309), errback=(continuation of Deferred@1420077921 after retry RPC after error@1793247907) -> (continuation of Deferred@2029455100 after retry RPC after error@1061966021) -> (continuation of Deferred@1312955374 after retry RPC after error@839481175) -> (continuation of Deferred@1913210636 after retry RPC after error@790559728) -> (continuation of Deferred@1910315158 after retry RPC after error@144144949) -> (continuation of Deferred@815688972 after retry RPC after error@1745342868) -> (continuation of Deferred@1896312234 after retry RPC after error@1123219309)))
21:03:46.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.797 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.798 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.939 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1131413378(state=PENDING, result=null, callback=(continuation of Deferred@428947313 after retry RPC after error@864729460) -> (continuation of Deferred@612507235 after retry RPC after error@414021278) -> (continuation of Deferred@135782469 after retry RPC after error@1857071402) -> (continuation of Deferred@1517671321 after retry RPC after error@689919951) -> (continuation of Deferred@507354977 after retry RPC after error@1966050421) -> (continuation of Deferred@28471378 after retry RPC after error@1379991172) -> (continuation of Deferred@552903994 after retry RPC after error@699517301), errback=(continuation of Deferred@428947313 after retry RPC after error@864729460) -> (continuation of Deferred@612507235 after retry RPC after error@414021278) -> (continuation of Deferred@135782469 after retry RPC after error@1857071402) -> (continuation of Deferred@1517671321 after retry RPC after error@689919951) -> (continuation of Deferred@507354977 after retry RPC after error@1966050421) -> (continuation of Deferred@28471378 after retry RPC after error@1379991172) -> (continuation of Deferred@552903994 after retry RPC after error@699517301)))
21:03:46.940 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1131413378(state=PENDING, result=null, callback=(continuation of Deferred@428947313 after retry RPC after error@864729460) -> (continuation of Deferred@612507235 after retry RPC after error@414021278) -> (continuation of Deferred@135782469 after retry RPC after error@1857071402) -> (continuation of Deferred@1517671321 after retry RPC after error@689919951) -> (continuation of Deferred@507354977 after retry RPC after error@1966050421) -> (continuation of Deferred@28471378 after retry RPC after error@1379991172) -> (continuation of Deferred@552903994 after retry RPC after error@699517301), errback=(continuation of Deferred@428947313 after retry RPC after error@864729460) -> (continuation of Deferred@612507235 after retry RPC after error@414021278) -> (continuation of Deferred@135782469 after retry RPC after error@1857071402) -> (continuation of Deferred@1517671321 after retry RPC after error@689919951) -> (continuation of Deferred@507354977 after retry RPC after error@1966050421) -> (continuation of Deferred@28471378 after retry RPC after error@1379991172) -> (continuation of Deferred@552903994 after retry RPC after error@699517301)))
21:03:46.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:46.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:46.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:46.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:46.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:46.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.060 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.199 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1498854913(state=PENDING, result=null, callback=(continuation of Deferred@1034689668 after retry RPC after error@1415936393) -> (continuation of Deferred@1606027403 after retry RPC after error@867191747) -> (continuation of Deferred@1847933197 after retry RPC after error@301186161) -> (continuation of Deferred@503447700 after retry RPC after error@794668740) -> (continuation of Deferred@1752635251 after retry RPC after error@1684745784) -> (continuation of Deferred@1793299585 after retry RPC after error@671759511) -> (continuation of Deferred@2023123562 after retry RPC after error@1160872366), errback=(continuation of Deferred@1034689668 after retry RPC after error@1415936393) -> (continuation of Deferred@1606027403 after retry RPC after error@867191747) -> (continuation of Deferred@1847933197 after retry RPC after error@301186161) -> (continuation of Deferred@503447700 after retry RPC after error@794668740) -> (continuation of Deferred@1752635251 after retry RPC after error@1684745784) -> (continuation of Deferred@1793299585 after retry RPC after error@671759511) -> (continuation of Deferred@2023123562 after retry RPC after error@1160872366)))
21:03:47.200 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1498854913(state=PENDING, result=null, callback=(continuation of Deferred@1034689668 after retry RPC after error@1415936393) -> (continuation of Deferred@1606027403 after retry RPC after error@867191747) -> (continuation of Deferred@1847933197 after retry RPC after error@301186161) -> (continuation of Deferred@503447700 after retry RPC after error@794668740) -> (continuation of Deferred@1752635251 after retry RPC after error@1684745784) -> (continuation of Deferred@1793299585 after retry RPC after error@671759511) -> (continuation of Deferred@2023123562 after retry RPC after error@1160872366), errback=(continuation of Deferred@1034689668 after retry RPC after error@1415936393) -> (continuation of Deferred@1606027403 after retry RPC after error@867191747) -> (continuation of Deferred@1847933197 after retry RPC after error@301186161) -> (continuation of Deferred@503447700 after retry RPC after error@794668740) -> (continuation of Deferred@1752635251 after retry RPC after error@1684745784) -> (continuation of Deferred@1793299585 after retry RPC after error@671759511) -> (continuation of Deferred@2023123562 after retry RPC after error@1160872366)))
21:03:47.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.460 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@851144963(state=PENDING, result=null, callback=(continuation of Deferred@1012423289 after retry RPC after error@83275503) -> (continuation of Deferred@1853963819 after retry RPC after error@123435409) -> (continuation of Deferred@2041471121 after retry RPC after error@1218328733) -> (continuation of Deferred@789133251 after retry RPC after error@1933726786) -> (continuation of Deferred@2044856403 after retry RPC after error@1082236194) -> (continuation of Deferred@642784734 after retry RPC after error@1196535058) -> (continuation of Deferred@773363003 after retry RPC after error@1656513879), errback=(continuation of Deferred@1012423289 after retry RPC after error@83275503) -> (continuation of Deferred@1853963819 after retry RPC after error@123435409) -> (continuation of Deferred@2041471121 after retry RPC after error@1218328733) -> (continuation of Deferred@789133251 after retry RPC after error@1933726786) -> (continuation of Deferred@2044856403 after retry RPC after error@1082236194) -> (continuation of Deferred@642784734 after retry RPC after error@1196535058) -> (continuation of Deferred@773363003 after retry RPC after error@1656513879)))
21:03:47.460 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@851144963(state=PENDING, result=null, callback=(continuation of Deferred@1012423289 after retry RPC after error@83275503) -> (continuation of Deferred@1853963819 after retry RPC after error@123435409) -> (continuation of Deferred@2041471121 after retry RPC after error@1218328733) -> (continuation of Deferred@789133251 after retry RPC after error@1933726786) -> (continuation of Deferred@2044856403 after retry RPC after error@1082236194) -> (continuation of Deferred@642784734 after retry RPC after error@1196535058) -> (continuation of Deferred@773363003 after retry RPC after error@1656513879), errback=(continuation of Deferred@1012423289 after retry RPC after error@83275503) -> (continuation of Deferred@1853963819 after retry RPC after error@123435409) -> (continuation of Deferred@2041471121 after retry RPC after error@1218328733) -> (continuation of Deferred@789133251 after retry RPC after error@1933726786) -> (continuation of Deferred@2044856403 after retry RPC after error@1082236194) -> (continuation of Deferred@642784734 after retry RPC after error@1196535058) -> (continuation of Deferred@773363003 after retry RPC after error@1656513879)))
21:03:47.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.497 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.497 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.599 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.600 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.640 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1295808432(state=PENDING, result=null, callback=(continuation of Deferred@1717123482 after retry RPC after error@1357522770) -> (continuation of Deferred@583157891 after retry RPC after error@208480924) -> (continuation of Deferred@601311074 after retry RPC after error@570832125) -> (continuation of Deferred@305198702 after retry RPC after error@2011957118) -> (continuation of Deferred@1847173814 after retry RPC after error@2100019796) -> (continuation of Deferred@1748879385 after retry RPC after error@2138961303), errback=(continuation of Deferred@1717123482 after retry RPC after error@1357522770) -> (continuation of Deferred@583157891 after retry RPC after error@208480924) -> (continuation of Deferred@601311074 after retry RPC after error@570832125) -> (continuation of Deferred@305198702 after retry RPC after error@2011957118) -> (continuation of Deferred@1847173814 after retry RPC after error@2100019796) -> (continuation of Deferred@1748879385 after retry RPC after error@2138961303)))
21:03:47.640 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1295808432(state=PENDING, result=null, callback=(continuation of Deferred@1717123482 after retry RPC after error@1357522770) -> (continuation of Deferred@583157891 after retry RPC after error@208480924) -> (continuation of Deferred@601311074 after retry RPC after error@570832125) -> (continuation of Deferred@305198702 after retry RPC after error@2011957118) -> (continuation of Deferred@1847173814 after retry RPC after error@2100019796) -> (continuation of Deferred@1748879385 after retry RPC after error@2138961303), errback=(continuation of Deferred@1717123482 after retry RPC after error@1357522770) -> (continuation of Deferred@583157891 after retry RPC after error@208480924) -> (continuation of Deferred@601311074 after retry RPC after error@570832125) -> (continuation of Deferred@305198702 after retry RPC after error@2011957118) -> (continuation of Deferred@1847173814 after retry RPC after error@2100019796) -> (continuation of Deferred@1748879385 after retry RPC after error@2138961303)))
21:03:47.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.699 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:47.735337 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:47.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:47.737735 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:47.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:47.739495 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:03:47.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.798 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.900 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1407145731(state=PENDING, result=null, callback=(continuation of Deferred@1227827606 after retry RPC after error@890584074) -> (continuation of Deferred@1941102038 after retry RPC after error@1526529004) -> (continuation of Deferred@1781467836 after retry RPC after error@1984335519) -> (continuation of Deferred@335524332 after retry RPC after error@778116068) -> (continuation of Deferred@1015647870 after retry RPC after error@2130304262) -> (continuation of Deferred@283599844 after retry RPC after error@914355239) -> (continuation of Deferred@1182099176 after retry RPC after error@1985254093), errback=(continuation of Deferred@1227827606 after retry RPC after error@890584074) -> (continuation of Deferred@1941102038 after retry RPC after error@1526529004) -> (continuation of Deferred@1781467836 after retry RPC after error@1984335519) -> (continuation of Deferred@335524332 after retry RPC after error@778116068) -> (continuation of Deferred@1015647870 after retry RPC after error@2130304262) -> (continuation of Deferred@283599844 after retry RPC after error@914355239) -> (continuation of Deferred@1182099176 after retry RPC after error@1985254093)))
21:03:47.901 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1407145731(state=PENDING, result=null, callback=(continuation of Deferred@1227827606 after retry RPC after error@890584074) -> (continuation of Deferred@1941102038 after retry RPC after error@1526529004) -> (continuation of Deferred@1781467836 after retry RPC after error@1984335519) -> (continuation of Deferred@335524332 after retry RPC after error@778116068) -> (continuation of Deferred@1015647870 after retry RPC after error@2130304262) -> (continuation of Deferred@283599844 after retry RPC after error@914355239) -> (continuation of Deferred@1182099176 after retry RPC after error@1985254093), errback=(continuation of Deferred@1227827606 after retry RPC after error@890584074) -> (continuation of Deferred@1941102038 after retry RPC after error@1526529004) -> (continuation of Deferred@1781467836 after retry RPC after error@1984335519) -> (continuation of Deferred@335524332 after retry RPC after error@778116068) -> (continuation of Deferred@1015647870 after retry RPC after error@2130304262) -> (continuation of Deferred@283599844 after retry RPC after error@914355239) -> (continuation of Deferred@1182099176 after retry RPC after error@1985254093)))
21:03:47.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:47.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:47.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:47.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:47.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:47.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.020 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.099 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@316416411(state=PENDING, result=null, callback=(continuation of Deferred@171098278 after retry RPC after error@463372570) -> (continuation of Deferred@719373952 after retry RPC after error@1506015934) -> (continuation of Deferred@1164457680 after retry RPC after error@714205494) -> (continuation of Deferred@115994146 after retry RPC after error@1085928011) -> (continuation of Deferred@1155764794 after retry RPC after error@1629359049) -> (continuation of Deferred@684683574 after retry RPC after error@661382606) -> (continuation of Deferred@2143408475 after retry RPC after error@1854983465), errback=(continuation of Deferred@171098278 after retry RPC after error@463372570) -> (continuation of Deferred@719373952 after retry RPC after error@1506015934) -> (continuation of Deferred@1164457680 after retry RPC after error@714205494) -> (continuation of Deferred@115994146 after retry RPC after error@1085928011) -> (continuation of Deferred@1155764794 after retry RPC after error@1629359049) -> (continuation of Deferred@684683574 after retry RPC after error@661382606) -> (continuation of Deferred@2143408475 after retry RPC after error@1854983465)))
21:03:48.101 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@316416411(state=PENDING, result=null, callback=(continuation of Deferred@171098278 after retry RPC after error@463372570) -> (continuation of Deferred@719373952 after retry RPC after error@1506015934) -> (continuation of Deferred@1164457680 after retry RPC after error@714205494) -> (continuation of Deferred@115994146 after retry RPC after error@1085928011) -> (continuation of Deferred@1155764794 after retry RPC after error@1629359049) -> (continuation of Deferred@684683574 after retry RPC after error@661382606) -> (continuation of Deferred@2143408475 after retry RPC after error@1854983465), errback=(continuation of Deferred@171098278 after retry RPC after error@463372570) -> (continuation of Deferred@719373952 after retry RPC after error@1506015934) -> (continuation of Deferred@1164457680 after retry RPC after error@714205494) -> (continuation of Deferred@115994146 after retry RPC after error@1085928011) -> (continuation of Deferred@1155764794 after retry RPC after error@1629359049) -> (continuation of Deferred@684683574 after retry RPC after error@661382606) -> (continuation of Deferred@2143408475 after retry RPC after error@1854983465)))
21:03:48.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.279 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.280 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.300 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@697690702(state=PENDING, result=null, callback=(continuation of Deferred@712783046 after retry RPC after error@1101227341) -> (continuation of Deferred@2001337556 after retry RPC after error@1904787433) -> (continuation of Deferred@983138074 after retry RPC after error@1615526921) -> (continuation of Deferred@1482490760 after retry RPC after error@1822333085) -> (continuation of Deferred@487480251 after retry RPC after error@714299047) -> (continuation of Deferred@451035454 after retry RPC after error@35682447) -> (continuation of Deferred@1830562035 after retry RPC after error@1560968601) -> (continuation of Deferred@1432960183 after retry RPC after error@1418515093), errback=(continuation of Deferred@712783046 after retry RPC after error@1101227341) -> (continuation of Deferred@2001337556 after retry RPC after error@1904787433) -> (continuation of Deferred@983138074 after retry RPC after error@1615526921) -> (continuation of Deferred@1482490760 after retry RPC after error@1822333085) -> (continuation of Deferred@487480251 after retry RPC after error@714299047) -> (continuation of Deferred@451035454 after retry RPC after error@35682447) -> (continuation of Deferred@1830562035 after retry RPC after error@1560968601) -> (continuation of Deferred@1432960183 after retry RPC after error@1418515093)))
21:03:48.300 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@697690702(state=PENDING, result=null, callback=(continuation of Deferred@712783046 after retry RPC after error@1101227341) -> (continuation of Deferred@2001337556 after retry RPC after error@1904787433) -> (continuation of Deferred@983138074 after retry RPC after error@1615526921) -> (continuation of Deferred@1482490760 after retry RPC after error@1822333085) -> (continuation of Deferred@487480251 after retry RPC after error@714299047) -> (continuation of Deferred@451035454 after retry RPC after error@35682447) -> (continuation of Deferred@1830562035 after retry RPC after error@1560968601) -> (continuation of Deferred@1432960183 after retry RPC after error@1418515093), errback=(continuation of Deferred@712783046 after retry RPC after error@1101227341) -> (continuation of Deferred@2001337556 after retry RPC after error@1904787433) -> (continuation of Deferred@983138074 after retry RPC after error@1615526921) -> (continuation of Deferred@1482490760 after retry RPC after error@1822333085) -> (continuation of Deferred@487480251 after retry RPC after error@714299047) -> (continuation of Deferred@451035454 after retry RPC after error@35682447) -> (continuation of Deferred@1830562035 after retry RPC after error@1560968601) -> (continuation of Deferred@1432960183 after retry RPC after error@1418515093)))
21:03:48.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.380 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.400 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.400 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.400 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.401 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.480 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1840606558(state=PENDING, result=null, callback=(continuation of Deferred@957100537 after retry RPC after error@1992260453) -> (continuation of Deferred@390910606 after retry RPC after error@1372951204) -> (continuation of Deferred@151970462 after retry RPC after error@82302089) -> (continuation of Deferred@1968094777 after retry RPC after error@150164102) -> (continuation of Deferred@1989630740 after retry RPC after error@1282312915) -> (continuation of Deferred@1280399889 after retry RPC after error@1587828351) -> (continuation of Deferred@304609604 after retry RPC after error@715641281), errback=(continuation of Deferred@957100537 after retry RPC after error@1992260453) -> (continuation of Deferred@390910606 after retry RPC after error@1372951204) -> (continuation of Deferred@151970462 after retry RPC after error@82302089) -> (continuation of Deferred@1968094777 after retry RPC after error@150164102) -> (continuation of Deferred@1989630740 after retry RPC after error@1282312915) -> (continuation of Deferred@1280399889 after retry RPC after error@1587828351) -> (continuation of Deferred@304609604 after retry RPC after error@715641281)))
21:03:48.480 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1840606558(state=PENDING, result=null, callback=(continuation of Deferred@957100537 after retry RPC after error@1992260453) -> (continuation of Deferred@390910606 after retry RPC after error@1372951204) -> (continuation of Deferred@151970462 after retry RPC after error@82302089) -> (continuation of Deferred@1968094777 after retry RPC after error@150164102) -> (continuation of Deferred@1989630740 after retry RPC after error@1282312915) -> (continuation of Deferred@1280399889 after retry RPC after error@1587828351) -> (continuation of Deferred@304609604 after retry RPC after error@715641281), errback=(continuation of Deferred@957100537 after retry RPC after error@1992260453) -> (continuation of Deferred@390910606 after retry RPC after error@1372951204) -> (continuation of Deferred@151970462 after retry RPC after error@82302089) -> (continuation of Deferred@1968094777 after retry RPC after error@150164102) -> (continuation of Deferred@1989630740 after retry RPC after error@1282312915) -> (continuation of Deferred@1280399889 after retry RPC after error@1587828351) -> (continuation of Deferred@304609604 after retry RPC after error@715641281)))
21:03:48.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.500 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.500 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.500 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.579 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.580 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.580 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.600 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.640 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2110849300(state=PENDING, result=null, callback=(continuation of Deferred@1027895496 after retry RPC after error@767081039) -> (continuation of Deferred@25621723 after retry RPC after error@1948945502) -> (continuation of Deferred@1794535459 after retry RPC after error@49599984) -> (continuation of Deferred@887282009 after retry RPC after error@344648679) -> (continuation of Deferred@1294267965 after retry RPC after error@1776017724) -> (continuation of Deferred@1272042831 after retry RPC after error@138926593), errback=(continuation of Deferred@1027895496 after retry RPC after error@767081039) -> (continuation of Deferred@25621723 after retry RPC after error@1948945502) -> (continuation of Deferred@1794535459 after retry RPC after error@49599984) -> (continuation of Deferred@887282009 after retry RPC after error@344648679) -> (continuation of Deferred@1294267965 after retry RPC after error@1776017724) -> (continuation of Deferred@1272042831 after retry RPC after error@138926593)))
21:03:48.640 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2110849300(state=PENDING, result=null, callback=(continuation of Deferred@1027895496 after retry RPC after error@767081039) -> (continuation of Deferred@25621723 after retry RPC after error@1948945502) -> (continuation of Deferred@1794535459 after retry RPC after error@49599984) -> (continuation of Deferred@887282009 after retry RPC after error@344648679) -> (continuation of Deferred@1294267965 after retry RPC after error@1776017724) -> (continuation of Deferred@1272042831 after retry RPC after error@138926593), errback=(continuation of Deferred@1027895496 after retry RPC after error@767081039) -> (continuation of Deferred@25621723 after retry RPC after error@1948945502) -> (continuation of Deferred@1794535459 after retry RPC after error@49599984) -> (continuation of Deferred@887282009 after retry RPC after error@344648679) -> (continuation of Deferred@1294267965 after retry RPC after error@1776017724) -> (continuation of Deferred@1272042831 after retry RPC after error@138926593)))
21:03:48.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.677 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.677 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.737 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.737 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.738 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.900 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1122466044(state=PENDING, result=null, callback=(continuation of Deferred@666045219 after retry RPC after error@1623317090) -> (continuation of Deferred@931164079 after retry RPC after error@1322897723) -> (continuation of Deferred@1896620496 after retry RPC after error@527497049) -> (continuation of Deferred@733552143 after retry RPC after error@728318538) -> (continuation of Deferred@997979222 after retry RPC after error@589895263) -> (continuation of Deferred@1082342739 after retry RPC after error@555306011) -> (continuation of Deferred@1168430378 after retry RPC after error@777300015), errback=(continuation of Deferred@666045219 after retry RPC after error@1623317090) -> (continuation of Deferred@931164079 after retry RPC after error@1322897723) -> (continuation of Deferred@1896620496 after retry RPC after error@527497049) -> (continuation of Deferred@733552143 after retry RPC after error@728318538) -> (continuation of Deferred@997979222 after retry RPC after error@589895263) -> (continuation of Deferred@1082342739 after retry RPC after error@555306011) -> (continuation of Deferred@1168430378 after retry RPC after error@777300015)))
21:03:48.901 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1122466044(state=PENDING, result=null, callback=(continuation of Deferred@666045219 after retry RPC after error@1623317090) -> (continuation of Deferred@931164079 after retry RPC after error@1322897723) -> (continuation of Deferred@1896620496 after retry RPC after error@527497049) -> (continuation of Deferred@733552143 after retry RPC after error@728318538) -> (continuation of Deferred@997979222 after retry RPC after error@589895263) -> (continuation of Deferred@1082342739 after retry RPC after error@555306011) -> (continuation of Deferred@1168430378 after retry RPC after error@777300015), errback=(continuation of Deferred@666045219 after retry RPC after error@1623317090) -> (continuation of Deferred@931164079 after retry RPC after error@1322897723) -> (continuation of Deferred@1896620496 after retry RPC after error@527497049) -> (continuation of Deferred@733552143 after retry RPC after error@728318538) -> (continuation of Deferred@997979222 after retry RPC after error@589895263) -> (continuation of Deferred@1082342739 after retry RPC after error@555306011) -> (continuation of Deferred@1168430378 after retry RPC after error@777300015)))
21:03:48.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.980 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:48.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:48.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:48.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:48.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:48.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.178 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1985323551(state=PENDING, result=null, callback=(continuation of Deferred@1264228954 after retry RPC after error@329515434) -> (continuation of Deferred@1260679175 after retry RPC after error@272266790) -> (continuation of Deferred@254649905 after retry RPC after error@1010261807) -> (continuation of Deferred@1417389286 after retry RPC after error@397632265) -> (continuation of Deferred@1976390624 after retry RPC after error@1929753681) -> (continuation of Deferred@2060481091 after retry RPC after error@1995959611) -> (continuation of Deferred@1773855318 after retry RPC after error@927167500), errback=(continuation of Deferred@1264228954 after retry RPC after error@329515434) -> (continuation of Deferred@1260679175 after retry RPC after error@272266790) -> (continuation of Deferred@254649905 after retry RPC after error@1010261807) -> (continuation of Deferred@1417389286 after retry RPC after error@397632265) -> (continuation of Deferred@1976390624 after retry RPC after error@1929753681) -> (continuation of Deferred@2060481091 after retry RPC after error@1995959611) -> (continuation of Deferred@1773855318 after retry RPC after error@927167500)))
21:03:49.178 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:49.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.257 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.258 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.359 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2138577161(state=PENDING, result=null, callback=(continuation of Deferred@29628423 after retry RPC after error@530804068) -> (continuation of Deferred@1039029275 after retry RPC after error@39957246) -> (continuation of Deferred@600086165 after retry RPC after error@654837700) -> (continuation of Deferred@1745473970 after retry RPC after error@1743812623) -> (continuation of Deferred@1611082330 after retry RPC after error@131479481) -> (continuation of Deferred@569696944 after retry RPC after error@1168168577) -> (continuation of Deferred@481063317 after retry RPC after error@720496198), errback=(continuation of Deferred@29628423 after retry RPC after error@530804068) -> (continuation of Deferred@1039029275 after retry RPC after error@39957246) -> (continuation of Deferred@600086165 after retry RPC after error@654837700) -> (continuation of Deferred@1745473970 after retry RPC after error@1743812623) -> (continuation of Deferred@1611082330 after retry RPC after error@131479481) -> (continuation of Deferred@569696944 after retry RPC after error@1168168577) -> (continuation of Deferred@481063317 after retry RPC after error@720496198)))
21:03:49.360 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2138577161(state=PENDING, result=null, callback=(continuation of Deferred@29628423 after retry RPC after error@530804068) -> (continuation of Deferred@1039029275 after retry RPC after error@39957246) -> (continuation of Deferred@600086165 after retry RPC after error@654837700) -> (continuation of Deferred@1745473970 after retry RPC after error@1743812623) -> (continuation of Deferred@1611082330 after retry RPC after error@131479481) -> (continuation of Deferred@569696944 after retry RPC after error@1168168577) -> (continuation of Deferred@481063317 after retry RPC after error@720496198), errback=(continuation of Deferred@29628423 after retry RPC after error@530804068) -> (continuation of Deferred@1039029275 after retry RPC after error@39957246) -> (continuation of Deferred@600086165 after retry RPC after error@654837700) -> (continuation of Deferred@1745473970 after retry RPC after error@1743812623) -> (continuation of Deferred@1611082330 after retry RPC after error@131479481) -> (continuation of Deferred@569696944 after retry RPC after error@1168168577) -> (continuation of Deferred@481063317 after retry RPC after error@720496198)))
21:03:49.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.419 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.620 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1309846414(state=PENDING, result=null, callback=(continuation of Deferred@1240261786 after retry RPC after error@789329979) -> (continuation of Deferred@631647412 after retry RPC after error@2117315925) -> (continuation of Deferred@1356160573 after retry RPC after error@517372412) -> (continuation of Deferred@176235806 after retry RPC after error@1765505054) -> (continuation of Deferred@333838258 after retry RPC after error@1053914539) -> (continuation of Deferred@1031474874 after retry RPC after error@263355011) -> (continuation of Deferred@869680288 after retry RPC after error@102699868), errback=(continuation of Deferred@1240261786 after retry RPC after error@789329979) -> (continuation of Deferred@631647412 after retry RPC after error@2117315925) -> (continuation of Deferred@1356160573 after retry RPC after error@517372412) -> (continuation of Deferred@176235806 after retry RPC after error@1765505054) -> (continuation of Deferred@333838258 after retry RPC after error@1053914539) -> (continuation of Deferred@1031474874 after retry RPC after error@263355011) -> (continuation of Deferred@869680288 after retry RPC after error@102699868)))
21:03:49.620 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1309846414(state=PENDING, result=null, callback=(continuation of Deferred@1240261786 after retry RPC after error@789329979) -> (continuation of Deferred@631647412 after retry RPC after error@2117315925) -> (continuation of Deferred@1356160573 after retry RPC after error@517372412) -> (continuation of Deferred@176235806 after retry RPC after error@1765505054) -> (continuation of Deferred@333838258 after retry RPC after error@1053914539) -> (continuation of Deferred@1031474874 after retry RPC after error@263355011) -> (continuation of Deferred@869680288 after retry RPC after error@102699868), errback=(continuation of Deferred@1240261786 after retry RPC after error@789329979) -> (continuation of Deferred@631647412 after retry RPC after error@2117315925) -> (continuation of Deferred@1356160573 after retry RPC after error@517372412) -> (continuation of Deferred@176235806 after retry RPC after error@1765505054) -> (continuation of Deferred@333838258 after retry RPC after error@1053914539) -> (continuation of Deferred@1031474874 after retry RPC after error@263355011) -> (continuation of Deferred@869680288 after retry RPC after error@102699868)))
21:03:49.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.659 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.660 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.679 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.819 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.819 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1937104638(state=PENDING, result=null, callback=(continuation of Deferred@880675835 after retry RPC after error@1691571200) -> (continuation of Deferred@844005163 after retry RPC after error@1863435698) -> (continuation of Deferred@729037432 after retry RPC after error@1442388034) -> (continuation of Deferred@344953585 after retry RPC after error@1968372103) -> (continuation of Deferred@2139077295 after retry RPC after error@1650034186) -> (continuation of Deferred@170494122 after retry RPC after error@1758844585), errback=(continuation of Deferred@880675835 after retry RPC after error@1691571200) -> (continuation of Deferred@844005163 after retry RPC after error@1863435698) -> (continuation of Deferred@729037432 after retry RPC after error@1442388034) -> (continuation of Deferred@344953585 after retry RPC after error@1968372103) -> (continuation of Deferred@2139077295 after retry RPC after error@1650034186) -> (continuation of Deferred@170494122 after retry RPC after error@1758844585)))
21:03:49.819 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1937104638(state=PENDING, result=null, callback=(continuation of Deferred@880675835 after retry RPC after error@1691571200) -> (continuation of Deferred@844005163 after retry RPC after error@1863435698) -> (continuation of Deferred@729037432 after retry RPC after error@1442388034) -> (continuation of Deferred@344953585 after retry RPC after error@1968372103) -> (continuation of Deferred@2139077295 after retry RPC after error@1650034186) -> (continuation of Deferred@170494122 after retry RPC after error@1758844585), errback=(continuation of Deferred@880675835 after retry RPC after error@1691571200) -> (continuation of Deferred@844005163 after retry RPC after error@1863435698) -> (continuation of Deferred@729037432 after retry RPC after error@1442388034) -> (continuation of Deferred@344953585 after retry RPC after error@1968372103) -> (continuation of Deferred@2139077295 after retry RPC after error@1650034186) -> (continuation of Deferred@170494122 after retry RPC after error@1758844585)))
21:03:49.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.919 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:49.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:49.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:49.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:49.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:49.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.020 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.021 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1911785633(state=PENDING, result=null, callback=(continuation of Deferred@356620400 after retry RPC after error@1071163016) -> (continuation of Deferred@1508833746 after retry RPC after error@875939022) -> (continuation of Deferred@766400544 after retry RPC after error@1463310331) -> (continuation of Deferred@2033587367 after retry RPC after error@1673510139) -> (continuation of Deferred@1016043508 after retry RPC after error@698189841) -> (continuation of Deferred@1623667369 after retry RPC after error@2122388014), errback=(continuation of Deferred@356620400 after retry RPC after error@1071163016) -> (continuation of Deferred@1508833746 after retry RPC after error@875939022) -> (continuation of Deferred@766400544 after retry RPC after error@1463310331) -> (continuation of Deferred@2033587367 after retry RPC after error@1673510139) -> (continuation of Deferred@1016043508 after retry RPC after error@698189841) -> (continuation of Deferred@1623667369 after retry RPC after error@2122388014)))
21:03:50.021 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1911785633(state=PENDING, result=null, callback=(continuation of Deferred@356620400 after retry RPC after error@1071163016) -> (continuation of Deferred@1508833746 after retry RPC after error@875939022) -> (continuation of Deferred@766400544 after retry RPC after error@1463310331) -> (continuation of Deferred@2033587367 after retry RPC after error@1673510139) -> (continuation of Deferred@1016043508 after retry RPC after error@698189841) -> (continuation of Deferred@1623667369 after retry RPC after error@2122388014), errback=(continuation of Deferred@356620400 after retry RPC after error@1071163016) -> (continuation of Deferred@1508833746 after retry RPC after error@875939022) -> (continuation of Deferred@766400544 after retry RPC after error@1463310331) -> (continuation of Deferred@2033587367 after retry RPC after error@1673510139) -> (continuation of Deferred@1016043508 after retry RPC after error@698189841) -> (continuation of Deferred@1623667369 after retry RPC after error@2122388014)))
21:03:50.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.058 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.260 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.260 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@492988678(state=PENDING, result=null, callback=(continuation of Deferred@319538158 after retry RPC after error@182457437) -> (continuation of Deferred@1869934621 after retry RPC after error@1641062060) -> (continuation of Deferred@738801099 after retry RPC after error@650223339) -> (continuation of Deferred@1853716022 after retry RPC after error@196976811) -> (continuation of Deferred@1862237050 after retry RPC after error@1155761800) -> (continuation of Deferred@1101694180 after retry RPC after error@661970050) -> (continuation of Deferred@851789757 after retry RPC after error@1783255211), errback=(continuation of Deferred@319538158 after retry RPC after error@182457437) -> (continuation of Deferred@1869934621 after retry RPC after error@1641062060) -> (continuation of Deferred@738801099 after retry RPC after error@650223339) -> (continuation of Deferred@1853716022 after retry RPC after error@196976811) -> (continuation of Deferred@1862237050 after retry RPC after error@1155761800) -> (continuation of Deferred@1101694180 after retry RPC after error@661970050) -> (continuation of Deferred@851789757 after retry RPC after error@1783255211)))
21:03:50.261 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@492988678(state=PENDING, result=null, callback=(continuation of Deferred@319538158 after retry RPC after error@182457437) -> (continuation of Deferred@1869934621 after retry RPC after error@1641062060) -> (continuation of Deferred@738801099 after retry RPC after error@650223339) -> (continuation of Deferred@1853716022 after retry RPC after error@196976811) -> (continuation of Deferred@1862237050 after retry RPC after error@1155761800) -> (continuation of Deferred@1101694180 after retry RPC after error@661970050) -> (continuation of Deferred@851789757 after retry RPC after error@1783255211), errback=(continuation of Deferred@319538158 after retry RPC after error@182457437) -> (continuation of Deferred@1869934621 after retry RPC after error@1641062060) -> (continuation of Deferred@738801099 after retry RPC after error@650223339) -> (continuation of Deferred@1853716022 after retry RPC after error@196976811) -> (continuation of Deferred@1862237050 after retry RPC after error@1155761800) -> (continuation of Deferred@1101694180 after retry RPC after error@661970050) -> (continuation of Deferred@851789757 after retry RPC after error@1783255211)))
21:03:50.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.460 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@384453847(state=PENDING, result=null, callback=(continuation of Deferred@1737910812 after retry RPC after error@1392994881) -> (continuation of Deferred@1767268352 after retry RPC after error@765334793) -> (continuation of Deferred@2030853164 after retry RPC after error@288449582) -> (continuation of Deferred@1270698088 after retry RPC after error@374743627) -> (continuation of Deferred@228651409 after retry RPC after error@388690975) -> (continuation of Deferred@909494402 after retry RPC after error@299294837) -> (continuation of Deferred@468760487 after retry RPC after error@1538929354) -> (continuation of Deferred@1137934567 after retry RPC after error@1704986016), errback=(continuation of Deferred@1737910812 after retry RPC after error@1392994881) -> (continuation of Deferred@1767268352 after retry RPC after error@765334793) -> (continuation of Deferred@2030853164 after retry RPC after error@288449582) -> (continuation of Deferred@1270698088 after retry RPC after error@374743627) -> (continuation of Deferred@228651409 after retry RPC after error@388690975) -> (continuation of Deferred@909494402 after retry RPC after error@299294837) -> (continuation of Deferred@468760487 after retry RPC after error@1538929354) -> (continuation of Deferred@1137934567 after retry RPC after error@1704986016)))
21:03:50.460 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@384453847(state=PENDING, result=null, callback=(continuation of Deferred@1737910812 after retry RPC after error@1392994881) -> (continuation of Deferred@1767268352 after retry RPC after error@765334793) -> (continuation of Deferred@2030853164 after retry RPC after error@288449582) -> (continuation of Deferred@1270698088 after retry RPC after error@374743627) -> (continuation of Deferred@228651409 after retry RPC after error@388690975) -> (continuation of Deferred@909494402 after retry RPC after error@299294837) -> (continuation of Deferred@468760487 after retry RPC after error@1538929354) -> (continuation of Deferred@1137934567 after retry RPC after error@1704986016), errback=(continuation of Deferred@1737910812 after retry RPC after error@1392994881) -> (continuation of Deferred@1767268352 after retry RPC after error@765334793) -> (continuation of Deferred@2030853164 after retry RPC after error@288449582) -> (continuation of Deferred@1270698088 after retry RPC after error@374743627) -> (continuation of Deferred@228651409 after retry RPC after error@388690975) -> (continuation of Deferred@909494402 after retry RPC after error@299294837) -> (continuation of Deferred@468760487 after retry RPC after error@1538929354) -> (continuation of Deferred@1137934567 after retry RPC after error@1704986016)))
21:03:50.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.659 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2033757548(state=PENDING, result=null, callback=(continuation of Deferred@1516042231 after retry RPC after error@2051099498) -> (continuation of Deferred@505446061 after retry RPC after error@575665096) -> (continuation of Deferred@521227065 after retry RPC after error@1878739370) -> (continuation of Deferred@1924372462 after retry RPC after error@801110979) -> (continuation of Deferred@979510670 after retry RPC after error@266440049) -> (continuation of Deferred@1627613060 after retry RPC after error@825424260), errback=(continuation of Deferred@1516042231 after retry RPC after error@2051099498) -> (continuation of Deferred@505446061 after retry RPC after error@575665096) -> (continuation of Deferred@521227065 after retry RPC after error@1878739370) -> (continuation of Deferred@1924372462 after retry RPC after error@801110979) -> (continuation of Deferred@979510670 after retry RPC after error@266440049) -> (continuation of Deferred@1627613060 after retry RPC after error@825424260)))
21:03:50.660 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2033757548(state=PENDING, result=null, callback=(continuation of Deferred@1516042231 after retry RPC after error@2051099498) -> (continuation of Deferred@505446061 after retry RPC after error@575665096) -> (continuation of Deferred@521227065 after retry RPC after error@1878739370) -> (continuation of Deferred@1924372462 after retry RPC after error@801110979) -> (continuation of Deferred@979510670 after retry RPC after error@266440049) -> (continuation of Deferred@1627613060 after retry RPC after error@825424260), errback=(continuation of Deferred@1516042231 after retry RPC after error@2051099498) -> (continuation of Deferred@505446061 after retry RPC after error@575665096) -> (continuation of Deferred@521227065 after retry RPC after error@1878739370) -> (continuation of Deferred@1924372462 after retry RPC after error@801110979) -> (continuation of Deferred@979510670 after retry RPC after error@266440049) -> (continuation of Deferred@1627613060 after retry RPC after error@825424260)))
21:03:50.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.680 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.739 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.840 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] sending RPC to server master-127.3.35.124:46649, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] refreshing cache from master, [19ms] sending RPC to server master-127.3.35.124:46649, [19ms] sending RPC to server master-127.3.35.125:42683, [19ms] sending RPC to server master-127.3.35.126:35753, [20ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] sending RPC to server master-127.3.35.124:46649, [39ms] sending RPC to server master-127.3.35.125:42683, [39ms] sending RPC to server master-127.3.35.126:35753, [40ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [60ms] sending RPC to server master-127.3.35.124:46649, [60ms] sending RPC to server master-127.3.35.125:42683, [60ms] sending RPC to server master-127.3.35.126:35753, [61ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] sending RPC to server master-127.3.35.124:46649, [80ms] sending RPC to server master-127.3.35.125:42683, [80ms] sending RPC to server master-127.3.35.126:35753, [80ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [99ms] refreshing cache from master, [99ms] sending RPC to server master-127.3.35.124:46649, [100ms] sending RPC to server master-127.3.35.125:42683, [100ms] sending RPC to server master-127.3.35.126:35753, [100ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] received response from server master-127.3.35.125:42683: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [159ms] refreshing cache from master, [159ms] sending RPC to server master-127.3.35.124:46649, [160ms] sending RPC to server master-127.3.35.125:42683, [160ms] sending RPC to server master-127.3.35.126:35753, [160ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1505711526(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.3.35.124:46649 -> (continuation of Deferred@459588460 after retry RPC after error@1429403326) -> (continuation of Deferred@1114874747 after retry RPC after error@1631771894) -> (continuation of Deferred@1551895569 after retry RPC after error@516265529) -> (continuation of Deferred@1098024673 after retry RPC after error@916477781) -> (continuation of Deferred@1788618004 after retry RPC after error@1490682400), errback=org.apache.kudu.client.ConnectToCluster$1@dcac254 -> ConnectToMasterErrCB for 127.3.35.124:46649 -> (continuation of Deferred@459588460 after retry RPC after error@1429403326) -> (continuation of Deferred@1114874747 after retry RPC after error@1631771894) -> (continuation of Deferred@1551895569 after retry RPC after error@516265529) -> (continuation of Deferred@1098024673 after retry RPC after error@916477781) -> (continuation of Deferred@1788618004 after retry RPC after error@1490682400)))
21:03:50.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] sending RPC to server master-127.3.35.124:46649, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] refreshing cache from master, [19ms] sending RPC to server master-127.3.35.124:46649, [19ms] sending RPC to server master-127.3.35.125:42683, [19ms] sending RPC to server master-127.3.35.126:35753, [20ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] sending RPC to server master-127.3.35.124:46649, [39ms] sending RPC to server master-127.3.35.125:42683, [39ms] sending RPC to server master-127.3.35.126:35753, [40ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [60ms] sending RPC to server master-127.3.35.124:46649, [60ms] sending RPC to server master-127.3.35.125:42683, [60ms] sending RPC to server master-127.3.35.126:35753, [61ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] sending RPC to server master-127.3.35.124:46649, [80ms] sending RPC to server master-127.3.35.125:42683, [80ms] sending RPC to server master-127.3.35.126:35753, [80ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [99ms] refreshing cache from master, [99ms] sending RPC to server master-127.3.35.124:46649, [100ms] sending RPC to server master-127.3.35.125:42683, [100ms] sending RPC to server master-127.3.35.126:35753, [100ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] received response from server master-127.3.35.125:42683: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [159ms] refreshing cache from master, [159ms] sending RPC to server master-127.3.35.124:46649, [160ms] sending RPC to server master-127.3.35.125:42683, [160ms] sending RPC to server master-127.3.35.126:35753, [160ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1505711526(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.3.35.124:46649 -> (continuation of Deferred@459588460 after retry RPC after error@1429403326) -> (continuation of Deferred@1114874747 after retry RPC after error@1631771894) -> (continuation of Deferred@1551895569 after retry RPC after error@516265529) -> (continuation of Deferred@1098024673 after retry RPC after error@916477781) -> (continuation of Deferred@1788618004 after retry RPC after error@1490682400), errback=org.apache.kudu.client.ConnectToCluster$1@dcac254 -> ConnectToMasterErrCB for 127.3.35.124:46649 -> (continuation of Deferred@459588460 after retry RPC after error@1429403326) -> (continuation of Deferred@1114874747 after retry RPC after error@1631771894) -> (continuation of Deferred@1551895569 after retry RPC after error@516265529) -> (continuation of Deferred@1098024673 after retry RPC after error@916477781) -> (continuation of Deferred@1788618004 after retry RPC after error@1490682400)))
21:03:50.841 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.917 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.938 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=5, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: delaying RPC due to: Illegal state: connection is terminated, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [19ms] Sub RPC ConnectToMaster: refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] Sub RPC ConnectToMaster: refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] Sub RPC ConnectToMaster: refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] Sub RPC ConnectToMaster: refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [99ms] Sub RPC ConnectToMaster: refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [101ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [159ms] Sub RPC ConnectToMaster: refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] sending RPC to server master-127.3.35.124:46649, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] refreshing cache from master, [19ms] sending RPC to server master-127.3.35.124:46649, [19ms] sending RPC to server master-127.3.35.125:42683, [19ms] sending RPC to server master-127.3.35.126:35753, [20ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] sending RPC to server master-127.3.35.124:46649, [39ms] sending RPC to server master-127.3.35.125:42683, [39ms] sending RPC to server master-127.3.35.126:35753, [40ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [60ms] sending RPC to server master-127.3.35.124:46649, [60ms] sending RPC to server master-127.3.35.125:42683, [60ms] sending RPC to server master-127.3.35.126:35753, [61ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] sending RPC to server master-127.3.35.124:46649, [80ms] sending RPC to server master-127.3.35.125:42683, [80ms] sending RPC to server master-127.3.35.126:35753, [80ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [99ms] refreshing cache from master, [99ms] sending RPC to server master-127.3.35.124:46649, [100ms] sending RPC to server master-127.3.35.125:42683, [100ms] sending RPC to server master-127.3.35.126:35753, [100ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] received response from server master-127.3.35.125:42683: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [159ms] refreshing cache from master, [159ms] sending RPC to server master-127.3.35.124:46649, [160ms] sending RPC to server master-127.3.35.125:42683, [160ms] sending RPC to server master-127.3.35.126:35753, [160ms] received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1505711526(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.3.35.124:46649 -> (continuation of Deferred@459588460 after retry RPC after error@1429403326) -> (continuation of Deferred@1114874747 after retry RPC after error@1631771894) -> (continuation of Deferred@1551895569 after retry RPC after error@516265529) -> (continuation of Deferred@1098024673 after retry RPC after error@916477781) -> (continuation of Deferred@1788618004 after retry RPC after error@1490682400), errback=org.apache.kudu.client.ConnectToCluster$1@dcac254 -> ConnectToMasterErrCB for 127.3.35.124:46649 -> (continuation of Deferred@459588460 after retry RPC after error@1429403326) -> (continuation of Deferred@1114874747 after retry RPC after error@1631771894) -> (continuation of Deferred@1551895569 after retry RPC after error@516265529) -> (continuation of Deferred@1098024673 after retry RPC after error@916477781) -> (continuation of Deferred@1788618004 after retry RPC after error@1490682400))), [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@629485836(state=PENDING, result=null, callback=(continuation of Deferred@1411325704 after retry RPC after error@950863077) -> (continuation of Deferred@581333451 after retry RPC after error@782711234) -> (continuation of Deferred@40670273 after retry RPC after error@1379279557) -> (continuation of Deferred@1133455713 after retry RPC after error@1416720891) -> (continuation of Deferred@844704168 after retry RPC after error@1369052460), errback=(continuation of Deferred@1411325704 after retry RPC after error@950863077) -> (continuation of Deferred@581333451 after retry RPC after error@782711234) -> (continuation of Deferred@40670273 after retry RPC after error@1379279557) -> (continuation of Deferred@1133455713 after retry RPC after error@1416720891) -> (continuation of Deferred@844704168 after retry RPC after error@1369052460)))
21:03:50.939 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:50.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:50.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:50.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:50.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:50.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:50.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.059 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.059 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.060 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.120 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@862615275(state=PENDING, result=null, callback=(continuation of Deferred@1190741627 after retry RPC after error@378127806) -> (continuation of Deferred@1298005757 after retry RPC after error@769660051) -> (continuation of Deferred@564549066 after retry RPC after error@2044379253) -> (continuation of Deferred@1532126117 after retry RPC after error@1992011984) -> (continuation of Deferred@1705030600 after retry RPC after error@1247153761) -> (continuation of Deferred@38863813 after retry RPC after error@700911215) -> (continuation of Deferred@1234818668 after retry RPC after error@860286008) -> (continuation of Deferred@964923070 after retry RPC after error@681991063), errback=(continuation of Deferred@1190741627 after retry RPC after error@378127806) -> (continuation of Deferred@1298005757 after retry RPC after error@769660051) -> (continuation of Deferred@564549066 after retry RPC after error@2044379253) -> (continuation of Deferred@1532126117 after retry RPC after error@1992011984) -> (continuation of Deferred@1705030600 after retry RPC after error@1247153761) -> (continuation of Deferred@38863813 after retry RPC after error@700911215) -> (continuation of Deferred@1234818668 after retry RPC after error@860286008) -> (continuation of Deferred@964923070 after retry RPC after error@681991063)))
21:03:51.120 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@862615275(state=PENDING, result=null, callback=(continuation of Deferred@1190741627 after retry RPC after error@378127806) -> (continuation of Deferred@1298005757 after retry RPC after error@769660051) -> (continuation of Deferred@564549066 after retry RPC after error@2044379253) -> (continuation of Deferred@1532126117 after retry RPC after error@1992011984) -> (continuation of Deferred@1705030600 after retry RPC after error@1247153761) -> (continuation of Deferred@38863813 after retry RPC after error@700911215) -> (continuation of Deferred@1234818668 after retry RPC after error@860286008) -> (continuation of Deferred@964923070 after retry RPC after error@681991063), errback=(continuation of Deferred@1190741627 after retry RPC after error@378127806) -> (continuation of Deferred@1298005757 after retry RPC after error@769660051) -> (continuation of Deferred@564549066 after retry RPC after error@2044379253) -> (continuation of Deferred@1532126117 after retry RPC after error@1992011984) -> (continuation of Deferred@1705030600 after retry RPC after error@1247153761) -> (continuation of Deferred@38863813 after retry RPC after error@700911215) -> (continuation of Deferred@1234818668 after retry RPC after error@860286008) -> (continuation of Deferred@964923070 after retry RPC after error@681991063)))
21:03:51.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.237 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.237 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.238 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.379 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1109639720(state=PENDING, result=null, callback=(continuation of Deferred@1039138542 after retry RPC after error@450815849) -> (continuation of Deferred@73016623 after retry RPC after error@1479841038) -> (continuation of Deferred@443418728 after retry RPC after error@2011578851) -> (continuation of Deferred@574644589 after retry RPC after error@1579158734) -> (continuation of Deferred@879190570 after retry RPC after error@1909183516) -> (continuation of Deferred@1344378945 after retry RPC after error@238663927) -> (continuation of Deferred@21032232 after retry RPC after error@772055592), errback=(continuation of Deferred@1039138542 after retry RPC after error@450815849) -> (continuation of Deferred@73016623 after retry RPC after error@1479841038) -> (continuation of Deferred@443418728 after retry RPC after error@2011578851) -> (continuation of Deferred@574644589 after retry RPC after error@1579158734) -> (continuation of Deferred@879190570 after retry RPC after error@1909183516) -> (continuation of Deferred@1344378945 after retry RPC after error@238663927) -> (continuation of Deferred@21032232 after retry RPC after error@772055592)))
21:03:51.380 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1109639720(state=PENDING, result=null, callback=(continuation of Deferred@1039138542 after retry RPC after error@450815849) -> (continuation of Deferred@73016623 after retry RPC after error@1479841038) -> (continuation of Deferred@443418728 after retry RPC after error@2011578851) -> (continuation of Deferred@574644589 after retry RPC after error@1579158734) -> (continuation of Deferred@879190570 after retry RPC after error@1909183516) -> (continuation of Deferred@1344378945 after retry RPC after error@238663927) -> (continuation of Deferred@21032232 after retry RPC after error@772055592), errback=(continuation of Deferred@1039138542 after retry RPC after error@450815849) -> (continuation of Deferred@73016623 after retry RPC after error@1479841038) -> (continuation of Deferred@443418728 after retry RPC after error@2011578851) -> (continuation of Deferred@574644589 after retry RPC after error@1579158734) -> (continuation of Deferred@879190570 after retry RPC after error@1909183516) -> (continuation of Deferred@1344378945 after retry RPC after error@238663927) -> (continuation of Deferred@21032232 after retry RPC after error@772055592)))
21:03:51.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.620 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.620 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@45086701(state=PENDING, result=null, callback=(continuation of Deferred@1302735308 after retry RPC after error@515480352) -> (continuation of Deferred@1594167597 after retry RPC after error@577275327) -> (continuation of Deferred@1585197175 after retry RPC after error@214876163) -> (continuation of Deferred@2075469006 after retry RPC after error@487986042) -> (continuation of Deferred@576091943 after retry RPC after error@1482843574) -> (continuation of Deferred@170113312 after retry RPC after error@662959837) -> (continuation of Deferred@922571333 after retry RPC after error@2105138937), errback=(continuation of Deferred@1302735308 after retry RPC after error@515480352) -> (continuation of Deferred@1594167597 after retry RPC after error@577275327) -> (continuation of Deferred@1585197175 after retry RPC after error@214876163) -> (continuation of Deferred@2075469006 after retry RPC after error@487986042) -> (continuation of Deferred@576091943 after retry RPC after error@1482843574) -> (continuation of Deferred@170113312 after retry RPC after error@662959837) -> (continuation of Deferred@922571333 after retry RPC after error@2105138937)))
21:03:51.621 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@45086701(state=PENDING, result=null, callback=(continuation of Deferred@1302735308 after retry RPC after error@515480352) -> (continuation of Deferred@1594167597 after retry RPC after error@577275327) -> (continuation of Deferred@1585197175 after retry RPC after error@214876163) -> (continuation of Deferred@2075469006 after retry RPC after error@487986042) -> (continuation of Deferred@576091943 after retry RPC after error@1482843574) -> (continuation of Deferred@170113312 after retry RPC after error@662959837) -> (continuation of Deferred@922571333 after retry RPC after error@2105138937), errback=(continuation of Deferred@1302735308 after retry RPC after error@515480352) -> (continuation of Deferred@1594167597 after retry RPC after error@577275327) -> (continuation of Deferred@1585197175 after retry RPC after error@214876163) -> (continuation of Deferred@2075469006 after retry RPC after error@487986042) -> (continuation of Deferred@576091943 after retry RPC after error@1482843574) -> (continuation of Deferred@170113312 after retry RPC after error@662959837) -> (continuation of Deferred@922571333 after retry RPC after error@2105138937)))
21:03:51.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.639 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.640 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.659 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.659 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.660 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.800 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@566189866(state=PENDING, result=null, callback=(continuation of Deferred@331580114 after retry RPC after error@1061509915) -> (continuation of Deferred@1545865537 after retry RPC after error@121195488) -> (continuation of Deferred@149130439 after retry RPC after error@501977923) -> (continuation of Deferred@1643769790 after retry RPC after error@743628330) -> (continuation of Deferred@952363731 after retry RPC after error@2037585667) -> (continuation of Deferred@1473876876 after retry RPC after error@1841341651) -> (continuation of Deferred@1249005258 after retry RPC after error@735609736), errback=(continuation of Deferred@331580114 after retry RPC after error@1061509915) -> (continuation of Deferred@1545865537 after retry RPC after error@121195488) -> (continuation of Deferred@149130439 after retry RPC after error@501977923) -> (continuation of Deferred@1643769790 after retry RPC after error@743628330) -> (continuation of Deferred@952363731 after retry RPC after error@2037585667) -> (continuation of Deferred@1473876876 after retry RPC after error@1841341651) -> (continuation of Deferred@1249005258 after retry RPC after error@735609736)))
21:03:51.800 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@566189866(state=PENDING, result=null, callback=(continuation of Deferred@331580114 after retry RPC after error@1061509915) -> (continuation of Deferred@1545865537 after retry RPC after error@121195488) -> (continuation of Deferred@149130439 after retry RPC after error@501977923) -> (continuation of Deferred@1643769790 after retry RPC after error@743628330) -> (continuation of Deferred@952363731 after retry RPC after error@2037585667) -> (continuation of Deferred@1473876876 after retry RPC after error@1841341651) -> (continuation of Deferred@1249005258 after retry RPC after error@735609736), errback=(continuation of Deferred@331580114 after retry RPC after error@1061509915) -> (continuation of Deferred@1545865537 after retry RPC after error@121195488) -> (continuation of Deferred@149130439 after retry RPC after error@501977923) -> (continuation of Deferred@1643769790 after retry RPC after error@743628330) -> (continuation of Deferred@952363731 after retry RPC after error@2037585667) -> (continuation of Deferred@1473876876 after retry RPC after error@1841341651) -> (continuation of Deferred@1249005258 after retry RPC after error@735609736)))
21:03:51.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:51.911190 4233 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:03:51.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:51.911432 4233 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:03:51.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:51.912261 4233 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:03:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:51.913188 3322 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:03:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:51.914331 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:03:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:51.914580 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:03:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:51.914669 3320 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:03:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:03:51.914904 4233 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:03:51.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:51.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:51.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:51.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:51.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:51.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.060 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1754381506(state=PENDING, result=null, callback=(continuation of Deferred@754213993 after retry RPC after error@299545245) -> (continuation of Deferred@1925210034 after retry RPC after error@391289476) -> (continuation of Deferred@411722539 after retry RPC after error@1543533821) -> (continuation of Deferred@751672576 after retry RPC after error@801678503) -> (continuation of Deferred@1678776654 after retry RPC after error@996687533) -> (continuation of Deferred@2064482653 after retry RPC after error@285933319) -> (continuation of Deferred@1974754817 after retry RPC after error@257133388), errback=(continuation of Deferred@754213993 after retry RPC after error@299545245) -> (continuation of Deferred@1925210034 after retry RPC after error@391289476) -> (continuation of Deferred@411722539 after retry RPC after error@1543533821) -> (continuation of Deferred@751672576 after retry RPC after error@801678503) -> (continuation of Deferred@1678776654 after retry RPC after error@996687533) -> (continuation of Deferred@2064482653 after retry RPC after error@285933319) -> (continuation of Deferred@1974754817 after retry RPC after error@257133388)))
21:03:52.060 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1754381506(state=PENDING, result=null, callback=(continuation of Deferred@754213993 after retry RPC after error@299545245) -> (continuation of Deferred@1925210034 after retry RPC after error@391289476) -> (continuation of Deferred@411722539 after retry RPC after error@1543533821) -> (continuation of Deferred@751672576 after retry RPC after error@801678503) -> (continuation of Deferred@1678776654 after retry RPC after error@996687533) -> (continuation of Deferred@2064482653 after retry RPC after error@285933319) -> (continuation of Deferred@1974754817 after retry RPC after error@257133388), errback=(continuation of Deferred@754213993 after retry RPC after error@299545245) -> (continuation of Deferred@1925210034 after retry RPC after error@391289476) -> (continuation of Deferred@411722539 after retry RPC after error@1543533821) -> (continuation of Deferred@751672576 after retry RPC after error@801678503) -> (continuation of Deferred@1678776654 after retry RPC after error@996687533) -> (continuation of Deferred@2064482653 after retry RPC after error@285933319) -> (continuation of Deferred@1974754817 after retry RPC after error@257133388)))
21:03:52.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.325 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@595955(state=PENDING, result=null, callback=(continuation of Deferred@1197603787 after retry RPC after error@1855733928) -> (continuation of Deferred@1793730625 after retry RPC after error@580131936) -> (continuation of Deferred@1784878170 after retry RPC after error@1719046334) -> (continuation of Deferred@1575293969 after retry RPC after error@1603302609) -> (continuation of Deferred@765688509 after retry RPC after error@2139290839) -> (continuation of Deferred@179448490 after retry RPC after error@1448665553) -> (continuation of Deferred@1620582197 after retry RPC after error@143930293), errback=(continuation of Deferred@1197603787 after retry RPC after error@1855733928) -> (continuation of Deferred@1793730625 after retry RPC after error@580131936) -> (continuation of Deferred@1784878170 after retry RPC after error@1719046334) -> (continuation of Deferred@1575293969 after retry RPC after error@1603302609) -> (continuation of Deferred@765688509 after retry RPC after error@2139290839) -> (continuation of Deferred@179448490 after retry RPC after error@1448665553) -> (continuation of Deferred@1620582197 after retry RPC after error@143930293)))
21:03:52.325 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@595955(state=PENDING, result=null, callback=(continuation of Deferred@1197603787 after retry RPC after error@1855733928) -> (continuation of Deferred@1793730625 after retry RPC after error@580131936) -> (continuation of Deferred@1784878170 after retry RPC after error@1719046334) -> (continuation of Deferred@1575293969 after retry RPC after error@1603302609) -> (continuation of Deferred@765688509 after retry RPC after error@2139290839) -> (continuation of Deferred@179448490 after retry RPC after error@1448665553) -> (continuation of Deferred@1620582197 after retry RPC after error@143930293), errback=(continuation of Deferred@1197603787 after retry RPC after error@1855733928) -> (continuation of Deferred@1793730625 after retry RPC after error@580131936) -> (continuation of Deferred@1784878170 after retry RPC after error@1719046334) -> (continuation of Deferred@1575293969 after retry RPC after error@1603302609) -> (continuation of Deferred@765688509 after retry RPC after error@2139290839) -> (continuation of Deferred@179448490 after retry RPC after error@1448665553) -> (continuation of Deferred@1620582197 after retry RPC after error@143930293)))
21:03:52.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.401 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.417 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.459 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.460 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.597 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1288700413(state=PENDING, result=null, callback=(continuation of Deferred@1057064009 after retry RPC after error@1729255764) -> (continuation of Deferred@941234197 after retry RPC after error@1946757378) -> (continuation of Deferred@1192876038 after retry RPC after error@842326344) -> (continuation of Deferred@1510378923 after retry RPC after error@1744498384) -> (continuation of Deferred@1848300481 after retry RPC after error@2072068589) -> (continuation of Deferred@352641251 after retry RPC after error@624427261) -> (continuation of Deferred@498367296 after retry RPC after error@1825296178), errback=(continuation of Deferred@1057064009 after retry RPC after error@1729255764) -> (continuation of Deferred@941234197 after retry RPC after error@1946757378) -> (continuation of Deferred@1192876038 after retry RPC after error@842326344) -> (continuation of Deferred@1510378923 after retry RPC after error@1744498384) -> (continuation of Deferred@1848300481 after retry RPC after error@2072068589) -> (continuation of Deferred@352641251 after retry RPC after error@624427261) -> (continuation of Deferred@498367296 after retry RPC after error@1825296178)))
21:03:52.597 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:52.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:52.756848 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:52.762279 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:52.763629 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:03:52.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.779 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@991178442(state=PENDING, result=null, callback=(continuation of Deferred@1845203389 after retry RPC after error@1788413958) -> (continuation of Deferred@910086723 after retry RPC after error@751224880) -> (continuation of Deferred@495047276 after retry RPC after error@1073357026) -> (continuation of Deferred@2140553620 after retry RPC after error@1703645470) -> (continuation of Deferred@1980382577 after retry RPC after error@404352610) -> (continuation of Deferred@1439602196 after retry RPC after error@1827995346), errback=(continuation of Deferred@1845203389 after retry RPC after error@1788413958) -> (continuation of Deferred@910086723 after retry RPC after error@751224880) -> (continuation of Deferred@495047276 after retry RPC after error@1073357026) -> (continuation of Deferred@2140553620 after retry RPC after error@1703645470) -> (continuation of Deferred@1980382577 after retry RPC after error@404352610) -> (continuation of Deferred@1439602196 after retry RPC after error@1827995346)))
21:03:52.779 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@991178442(state=PENDING, result=null, callback=(continuation of Deferred@1845203389 after retry RPC after error@1788413958) -> (continuation of Deferred@910086723 after retry RPC after error@751224880) -> (continuation of Deferred@495047276 after retry RPC after error@1073357026) -> (continuation of Deferred@2140553620 after retry RPC after error@1703645470) -> (continuation of Deferred@1980382577 after retry RPC after error@404352610) -> (continuation of Deferred@1439602196 after retry RPC after error@1827995346), errback=(continuation of Deferred@1845203389 after retry RPC after error@1788413958) -> (continuation of Deferred@910086723 after retry RPC after error@751224880) -> (continuation of Deferred@495047276 after retry RPC after error@1073357026) -> (continuation of Deferred@2140553620 after retry RPC after error@1703645470) -> (continuation of Deferred@1980382577 after retry RPC after error@404352610) -> (continuation of Deferred@1439602196 after retry RPC after error@1827995346)))
21:03:52.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.799 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.959 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.960 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@205639969(state=PENDING, result=null, callback=(continuation of Deferred@1120246695 after retry RPC after error@1567489146) -> (continuation of Deferred@1438420196 after retry RPC after error@1475774379) -> (continuation of Deferred@956315217 after retry RPC after error@196903719) -> (continuation of Deferred@978773272 after retry RPC after error@1501488678) -> (continuation of Deferred@1635049958 after retry RPC after error@247391060) -> (continuation of Deferred@63809982 after retry RPC after error@757866551), errback=(continuation of Deferred@1120246695 after retry RPC after error@1567489146) -> (continuation of Deferred@1438420196 after retry RPC after error@1475774379) -> (continuation of Deferred@956315217 after retry RPC after error@196903719) -> (continuation of Deferred@978773272 after retry RPC after error@1501488678) -> (continuation of Deferred@1635049958 after retry RPC after error@247391060) -> (continuation of Deferred@63809982 after retry RPC after error@757866551)))
21:03:52.961 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@205639969(state=PENDING, result=null, callback=(continuation of Deferred@1120246695 after retry RPC after error@1567489146) -> (continuation of Deferred@1438420196 after retry RPC after error@1475774379) -> (continuation of Deferred@956315217 after retry RPC after error@196903719) -> (continuation of Deferred@978773272 after retry RPC after error@1501488678) -> (continuation of Deferred@1635049958 after retry RPC after error@247391060) -> (continuation of Deferred@63809982 after retry RPC after error@757866551), errback=(continuation of Deferred@1120246695 after retry RPC after error@1567489146) -> (continuation of Deferred@1438420196 after retry RPC after error@1475774379) -> (continuation of Deferred@956315217 after retry RPC after error@196903719) -> (continuation of Deferred@978773272 after retry RPC after error@1501488678) -> (continuation of Deferred@1635049958 after retry RPC after error@247391060) -> (continuation of Deferred@63809982 after retry RPC after error@757866551)))
21:03:52.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.980 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:52.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:52.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:52.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:52.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:52.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.179 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@202315549(state=PENDING, result=null, callback=(continuation of Deferred@1220538041 after retry RPC after error@1800457458) -> (continuation of Deferred@1908275007 after retry RPC after error@100577770) -> (continuation of Deferred@847465930 after retry RPC after error@1599676468) -> (continuation of Deferred@1103311087 after retry RPC after error@834840796) -> (continuation of Deferred@419913267 after retry RPC after error@256476583) -> (continuation of Deferred@1465223401 after retry RPC after error@1945456279) -> (continuation of Deferred@1344303132 after retry RPC after error@319267586), errback=(continuation of Deferred@1220538041 after retry RPC after error@1800457458) -> (continuation of Deferred@1908275007 after retry RPC after error@100577770) -> (continuation of Deferred@847465930 after retry RPC after error@1599676468) -> (continuation of Deferred@1103311087 after retry RPC after error@834840796) -> (continuation of Deferred@419913267 after retry RPC after error@256476583) -> (continuation of Deferred@1465223401 after retry RPC after error@1945456279) -> (continuation of Deferred@1344303132 after retry RPC after error@319267586)))
21:03:53.180 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@202315549(state=PENDING, result=null, callback=(continuation of Deferred@1220538041 after retry RPC after error@1800457458) -> (continuation of Deferred@1908275007 after retry RPC after error@100577770) -> (continuation of Deferred@847465930 after retry RPC after error@1599676468) -> (continuation of Deferred@1103311087 after retry RPC after error@834840796) -> (continuation of Deferred@419913267 after retry RPC after error@256476583) -> (continuation of Deferred@1465223401 after retry RPC after error@1945456279) -> (continuation of Deferred@1344303132 after retry RPC after error@319267586), errback=(continuation of Deferred@1220538041 after retry RPC after error@1800457458) -> (continuation of Deferred@1908275007 after retry RPC after error@100577770) -> (continuation of Deferred@847465930 after retry RPC after error@1599676468) -> (continuation of Deferred@1103311087 after retry RPC after error@834840796) -> (continuation of Deferred@419913267 after retry RPC after error@256476583) -> (continuation of Deferred@1465223401 after retry RPC after error@1945456279) -> (continuation of Deferred@1344303132 after retry RPC after error@319267586)))
21:03:53.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.359 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@250029847(state=PENDING, result=null, callback=(continuation of Deferred@1186442145 after retry RPC after error@507341731) -> (continuation of Deferred@1614684454 after retry RPC after error@1465286057) -> (continuation of Deferred@742866497 after retry RPC after error@1498560537) -> (continuation of Deferred@1215563012 after retry RPC after error@741053078) -> (continuation of Deferred@957027428 after retry RPC after error@1909547436) -> (continuation of Deferred@434439863 after retry RPC after error@1764395071), errback=(continuation of Deferred@1186442145 after retry RPC after error@507341731) -> (continuation of Deferred@1614684454 after retry RPC after error@1465286057) -> (continuation of Deferred@742866497 after retry RPC after error@1498560537) -> (continuation of Deferred@1215563012 after retry RPC after error@741053078) -> (continuation of Deferred@957027428 after retry RPC after error@1909547436) -> (continuation of Deferred@434439863 after retry RPC after error@1764395071)))
21:03:53.360 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@250029847(state=PENDING, result=null, callback=(continuation of Deferred@1186442145 after retry RPC after error@507341731) -> (continuation of Deferred@1614684454 after retry RPC after error@1465286057) -> (continuation of Deferred@742866497 after retry RPC after error@1498560537) -> (continuation of Deferred@1215563012 after retry RPC after error@741053078) -> (continuation of Deferred@957027428 after retry RPC after error@1909547436) -> (continuation of Deferred@434439863 after retry RPC after error@1764395071), errback=(continuation of Deferred@1186442145 after retry RPC after error@507341731) -> (continuation of Deferred@1614684454 after retry RPC after error@1465286057) -> (continuation of Deferred@742866497 after retry RPC after error@1498560537) -> (continuation of Deferred@1215563012 after retry RPC after error@741053078) -> (continuation of Deferred@957027428 after retry RPC after error@1909547436) -> (continuation of Deferred@434439863 after retry RPC after error@1764395071)))
21:03:53.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.380 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.540 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.540 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@91622390(state=PENDING, result=null, callback=(continuation of Deferred@1961112040 after retry RPC after error@745891905) -> (continuation of Deferred@38924159 after retry RPC after error@936209784) -> (continuation of Deferred@1651732760 after retry RPC after error@1698904695) -> (continuation of Deferred@1751391159 after retry RPC after error@882773358) -> (continuation of Deferred@1293701460 after retry RPC after error@829446658) -> (continuation of Deferred@2030126648 after retry RPC after error@1034998027) -> (continuation of Deferred@302620952 after retry RPC after error@555374865), errback=(continuation of Deferred@1961112040 after retry RPC after error@745891905) -> (continuation of Deferred@38924159 after retry RPC after error@936209784) -> (continuation of Deferred@1651732760 after retry RPC after error@1698904695) -> (continuation of Deferred@1751391159 after retry RPC after error@882773358) -> (continuation of Deferred@1293701460 after retry RPC after error@829446658) -> (continuation of Deferred@2030126648 after retry RPC after error@1034998027) -> (continuation of Deferred@302620952 after retry RPC after error@555374865)))
21:03:53.541 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@91622390(state=PENDING, result=null, callback=(continuation of Deferred@1961112040 after retry RPC after error@745891905) -> (continuation of Deferred@38924159 after retry RPC after error@936209784) -> (continuation of Deferred@1651732760 after retry RPC after error@1698904695) -> (continuation of Deferred@1751391159 after retry RPC after error@882773358) -> (continuation of Deferred@1293701460 after retry RPC after error@829446658) -> (continuation of Deferred@2030126648 after retry RPC after error@1034998027) -> (continuation of Deferred@302620952 after retry RPC after error@555374865), errback=(continuation of Deferred@1961112040 after retry RPC after error@745891905) -> (continuation of Deferred@38924159 after retry RPC after error@936209784) -> (continuation of Deferred@1651732760 after retry RPC after error@1698904695) -> (continuation of Deferred@1751391159 after retry RPC after error@882773358) -> (continuation of Deferred@1293701460 after retry RPC after error@829446658) -> (continuation of Deferred@2030126648 after retry RPC after error@1034998027) -> (continuation of Deferred@302620952 after retry RPC after error@555374865)))
21:03:53.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.660 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.740 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1609132522(state=PENDING, result=null, callback=(continuation of Deferred@1924747501 after retry RPC after error@1438097010) -> (continuation of Deferred@1925043753 after retry RPC after error@1712281517) -> (continuation of Deferred@1586056633 after retry RPC after error@933497876) -> (continuation of Deferred@733255767 after retry RPC after error@932843699) -> (continuation of Deferred@594175868 after retry RPC after error@884546473) -> (continuation of Deferred@994073417 after retry RPC after error@1547225882) -> (continuation of Deferred@748230246 after retry RPC after error@1350644751), errback=(continuation of Deferred@1924747501 after retry RPC after error@1438097010) -> (continuation of Deferred@1925043753 after retry RPC after error@1712281517) -> (continuation of Deferred@1586056633 after retry RPC after error@933497876) -> (continuation of Deferred@733255767 after retry RPC after error@932843699) -> (continuation of Deferred@594175868 after retry RPC after error@884546473) -> (continuation of Deferred@994073417 after retry RPC after error@1547225882) -> (continuation of Deferred@748230246 after retry RPC after error@1350644751)))
21:03:53.740 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1609132522(state=PENDING, result=null, callback=(continuation of Deferred@1924747501 after retry RPC after error@1438097010) -> (continuation of Deferred@1925043753 after retry RPC after error@1712281517) -> (continuation of Deferred@1586056633 after retry RPC after error@933497876) -> (continuation of Deferred@733255767 after retry RPC after error@932843699) -> (continuation of Deferred@594175868 after retry RPC after error@884546473) -> (continuation of Deferred@994073417 after retry RPC after error@1547225882) -> (continuation of Deferred@748230246 after retry RPC after error@1350644751), errback=(continuation of Deferred@1924747501 after retry RPC after error@1438097010) -> (continuation of Deferred@1925043753 after retry RPC after error@1712281517) -> (continuation of Deferred@1586056633 after retry RPC after error@933497876) -> (continuation of Deferred@733255767 after retry RPC after error@932843699) -> (continuation of Deferred@594175868 after retry RPC after error@884546473) -> (continuation of Deferred@994073417 after retry RPC after error@1547225882) -> (continuation of Deferred@748230246 after retry RPC after error@1350644751)))
21:03:53.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.777 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.980 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:53.981 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2037031000(state=PENDING, result=null, callback=(continuation of Deferred@134303182 after retry RPC after error@1362514155) -> (continuation of Deferred@826016554 after retry RPC after error@502799669) -> (continuation of Deferred@521489529 after retry RPC after error@2093475448) -> (continuation of Deferred@766742425 after retry RPC after error@1887387311) -> (continuation of Deferred@1161609821 after retry RPC after error@101390500) -> (continuation of Deferred@938836155 after retry RPC after error@1056332460) -> (continuation of Deferred@494622981 after retry RPC after error@1131134006), errback=(continuation of Deferred@134303182 after retry RPC after error@1362514155) -> (continuation of Deferred@826016554 after retry RPC after error@502799669) -> (continuation of Deferred@521489529 after retry RPC after error@2093475448) -> (continuation of Deferred@766742425 after retry RPC after error@1887387311) -> (continuation of Deferred@1161609821 after retry RPC after error@101390500) -> (continuation of Deferred@938836155 after retry RPC after error@1056332460) -> (continuation of Deferred@494622981 after retry RPC after error@1131134006)))
21:03:53.981 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2037031000(state=PENDING, result=null, callback=(continuation of Deferred@134303182 after retry RPC after error@1362514155) -> (continuation of Deferred@826016554 after retry RPC after error@502799669) -> (continuation of Deferred@521489529 after retry RPC after error@2093475448) -> (continuation of Deferred@766742425 after retry RPC after error@1887387311) -> (continuation of Deferred@1161609821 after retry RPC after error@101390500) -> (continuation of Deferred@938836155 after retry RPC after error@1056332460) -> (continuation of Deferred@494622981 after retry RPC after error@1131134006), errback=(continuation of Deferred@134303182 after retry RPC after error@1362514155) -> (continuation of Deferred@826016554 after retry RPC after error@502799669) -> (continuation of Deferred@521489529 after retry RPC after error@2093475448) -> (continuation of Deferred@766742425 after retry RPC after error@1887387311) -> (continuation of Deferred@1161609821 after retry RPC after error@101390500) -> (continuation of Deferred@938836155 after retry RPC after error@1056332460) -> (continuation of Deferred@494622981 after retry RPC after error@1131134006)))
21:03:53.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:53.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:53.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:53.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:53.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.097 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.097 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.200 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1381689006(state=PENDING, result=null, callback=(continuation of Deferred@2003844051 after retry RPC after error@954606764) -> (continuation of Deferred@1029398793 after retry RPC after error@951160730) -> (continuation of Deferred@1315538428 after retry RPC after error@1201867737) -> (continuation of Deferred@563509517 after retry RPC after error@368046074) -> (continuation of Deferred@397139678 after retry RPC after error@1208611377) -> (continuation of Deferred@1522902060 after retry RPC after error@817823921) -> (continuation of Deferred@2032887126 after retry RPC after error@2117371960), errback=(continuation of Deferred@2003844051 after retry RPC after error@954606764) -> (continuation of Deferred@1029398793 after retry RPC after error@951160730) -> (continuation of Deferred@1315538428 after retry RPC after error@1201867737) -> (continuation of Deferred@563509517 after retry RPC after error@368046074) -> (continuation of Deferred@397139678 after retry RPC after error@1208611377) -> (continuation of Deferred@1522902060 after retry RPC after error@817823921) -> (continuation of Deferred@2032887126 after retry RPC after error@2117371960)))
21:03:54.201 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1381689006(state=PENDING, result=null, callback=(continuation of Deferred@2003844051 after retry RPC after error@954606764) -> (continuation of Deferred@1029398793 after retry RPC after error@951160730) -> (continuation of Deferred@1315538428 after retry RPC after error@1201867737) -> (continuation of Deferred@563509517 after retry RPC after error@368046074) -> (continuation of Deferred@397139678 after retry RPC after error@1208611377) -> (continuation of Deferred@1522902060 after retry RPC after error@817823921) -> (continuation of Deferred@2032887126 after retry RPC after error@2117371960), errback=(continuation of Deferred@2003844051 after retry RPC after error@954606764) -> (continuation of Deferred@1029398793 after retry RPC after error@951160730) -> (continuation of Deferred@1315538428 after retry RPC after error@1201867737) -> (continuation of Deferred@563509517 after retry RPC after error@368046074) -> (continuation of Deferred@397139678 after retry RPC after error@1208611377) -> (continuation of Deferred@1522902060 after retry RPC after error@817823921) -> (continuation of Deferred@2032887126 after retry RPC after error@2117371960)))
21:03:54.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.239 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.400 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@347043079(state=PENDING, result=null, callback=(continuation of Deferred@365180124 after retry RPC after error@649556793) -> (continuation of Deferred@1378313034 after retry RPC after error@1028600644) -> (continuation of Deferred@161059557 after retry RPC after error@338298452) -> (continuation of Deferred@2113804968 after retry RPC after error@972018496) -> (continuation of Deferred@2082762786 after retry RPC after error@955826516) -> (continuation of Deferred@895395769 after retry RPC after error@1928071041), errback=(continuation of Deferred@365180124 after retry RPC after error@649556793) -> (continuation of Deferred@1378313034 after retry RPC after error@1028600644) -> (continuation of Deferred@161059557 after retry RPC after error@338298452) -> (continuation of Deferred@2113804968 after retry RPC after error@972018496) -> (continuation of Deferred@2082762786 after retry RPC after error@955826516) -> (continuation of Deferred@895395769 after retry RPC after error@1928071041)))
21:03:54.400 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@347043079(state=PENDING, result=null, callback=(continuation of Deferred@365180124 after retry RPC after error@649556793) -> (continuation of Deferred@1378313034 after retry RPC after error@1028600644) -> (continuation of Deferred@161059557 after retry RPC after error@338298452) -> (continuation of Deferred@2113804968 after retry RPC after error@972018496) -> (continuation of Deferred@2082762786 after retry RPC after error@955826516) -> (continuation of Deferred@895395769 after retry RPC after error@1928071041), errback=(continuation of Deferred@365180124 after retry RPC after error@649556793) -> (continuation of Deferred@1378313034 after retry RPC after error@1028600644) -> (continuation of Deferred@161059557 after retry RPC after error@338298452) -> (continuation of Deferred@2113804968 after retry RPC after error@972018496) -> (continuation of Deferred@2082762786 after retry RPC after error@955826516) -> (continuation of Deferred@895395769 after retry RPC after error@1928071041)))
21:03:54.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.540 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.660 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@516389359(state=PENDING, result=null, callback=(continuation of Deferred@193081225 after retry RPC after error@2133859428) -> (continuation of Deferred@1311946039 after retry RPC after error@1797240095) -> (continuation of Deferred@1903894930 after retry RPC after error@249024741) -> (continuation of Deferred@1435981573 after retry RPC after error@1136044005) -> (continuation of Deferred@1810463152 after retry RPC after error@1535757520) -> (continuation of Deferred@877234139 after retry RPC after error@1119258424) -> (continuation of Deferred@1647464999 after retry RPC after error@2001268221), errback=(continuation of Deferred@193081225 after retry RPC after error@2133859428) -> (continuation of Deferred@1311946039 after retry RPC after error@1797240095) -> (continuation of Deferred@1903894930 after retry RPC after error@249024741) -> (continuation of Deferred@1435981573 after retry RPC after error@1136044005) -> (continuation of Deferred@1810463152 after retry RPC after error@1535757520) -> (continuation of Deferred@877234139 after retry RPC after error@1119258424) -> (continuation of Deferred@1647464999 after retry RPC after error@2001268221)))
21:03:54.661 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@516389359(state=PENDING, result=null, callback=(continuation of Deferred@193081225 after retry RPC after error@2133859428) -> (continuation of Deferred@1311946039 after retry RPC after error@1797240095) -> (continuation of Deferred@1903894930 after retry RPC after error@249024741) -> (continuation of Deferred@1435981573 after retry RPC after error@1136044005) -> (continuation of Deferred@1810463152 after retry RPC after error@1535757520) -> (continuation of Deferred@877234139 after retry RPC after error@1119258424) -> (continuation of Deferred@1647464999 after retry RPC after error@2001268221), errback=(continuation of Deferred@193081225 after retry RPC after error@2133859428) -> (continuation of Deferred@1311946039 after retry RPC after error@1797240095) -> (continuation of Deferred@1903894930 after retry RPC after error@249024741) -> (continuation of Deferred@1435981573 after retry RPC after error@1136044005) -> (continuation of Deferred@1810463152 after retry RPC after error@1535757520) -> (continuation of Deferred@877234139 after retry RPC after error@1119258424) -> (continuation of Deferred@1647464999 after retry RPC after error@2001268221)))
21:03:54.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.697 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.697 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.698 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.900 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2100899147(state=PENDING, result=null, callback=(continuation of Deferred@1726971714 after retry RPC after error@535083861) -> (continuation of Deferred@195362112 after retry RPC after error@1069799946) -> (continuation of Deferred@575383519 after retry RPC after error@416259792) -> (continuation of Deferred@883764852 after retry RPC after error@362787169) -> (continuation of Deferred@1779800402 after retry RPC after error@239641566) -> (continuation of Deferred@1405870829 after retry RPC after error@993482060) -> (continuation of Deferred@2108589169 after retry RPC after error@909348425), errback=(continuation of Deferred@1726971714 after retry RPC after error@535083861) -> (continuation of Deferred@195362112 after retry RPC after error@1069799946) -> (continuation of Deferred@575383519 after retry RPC after error@416259792) -> (continuation of Deferred@883764852 after retry RPC after error@362787169) -> (continuation of Deferred@1779800402 after retry RPC after error@239641566) -> (continuation of Deferred@1405870829 after retry RPC after error@993482060) -> (continuation of Deferred@2108589169 after retry RPC after error@909348425)))
21:03:54.900 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2100899147(state=PENDING, result=null, callback=(continuation of Deferred@1726971714 after retry RPC after error@535083861) -> (continuation of Deferred@195362112 after retry RPC after error@1069799946) -> (continuation of Deferred@575383519 after retry RPC after error@416259792) -> (continuation of Deferred@883764852 after retry RPC after error@362787169) -> (continuation of Deferred@1779800402 after retry RPC after error@239641566) -> (continuation of Deferred@1405870829 after retry RPC after error@993482060) -> (continuation of Deferred@2108589169 after retry RPC after error@909348425), errback=(continuation of Deferred@1726971714 after retry RPC after error@535083861) -> (continuation of Deferred@195362112 after retry RPC after error@1069799946) -> (continuation of Deferred@575383519 after retry RPC after error@416259792) -> (continuation of Deferred@883764852 after retry RPC after error@362787169) -> (continuation of Deferred@1779800402 after retry RPC after error@239641566) -> (continuation of Deferred@1405870829 after retry RPC after error@993482060) -> (continuation of Deferred@2108589169 after retry RPC after error@909348425)))
21:03:54.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:54.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:54.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:54.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:54.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:54.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.079 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@447756190(state=PENDING, result=null, callback=(continuation of Deferred@1005826353 after retry RPC after error@1385072398) -> (continuation of Deferred@1384501864 after retry RPC after error@1234309985) -> (continuation of Deferred@1875794267 after retry RPC after error@1239291227) -> (continuation of Deferred@901023898 after retry RPC after error@1818528854) -> (continuation of Deferred@1904018432 after retry RPC after error@1211151289) -> (continuation of Deferred@1506873981 after retry RPC after error@295641901) -> (continuation of Deferred@140212250 after retry RPC after error@1161543635), errback=(continuation of Deferred@1005826353 after retry RPC after error@1385072398) -> (continuation of Deferred@1384501864 after retry RPC after error@1234309985) -> (continuation of Deferred@1875794267 after retry RPC after error@1239291227) -> (continuation of Deferred@901023898 after retry RPC after error@1818528854) -> (continuation of Deferred@1904018432 after retry RPC after error@1211151289) -> (continuation of Deferred@1506873981 after retry RPC after error@295641901) -> (continuation of Deferred@140212250 after retry RPC after error@1161543635)))
21:03:55.080 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@447756190(state=PENDING, result=null, callback=(continuation of Deferred@1005826353 after retry RPC after error@1385072398) -> (continuation of Deferred@1384501864 after retry RPC after error@1234309985) -> (continuation of Deferred@1875794267 after retry RPC after error@1239291227) -> (continuation of Deferred@901023898 after retry RPC after error@1818528854) -> (continuation of Deferred@1904018432 after retry RPC after error@1211151289) -> (continuation of Deferred@1506873981 after retry RPC after error@295641901) -> (continuation of Deferred@140212250 after retry RPC after error@1161543635), errback=(continuation of Deferred@1005826353 after retry RPC after error@1385072398) -> (continuation of Deferred@1384501864 after retry RPC after error@1234309985) -> (continuation of Deferred@1875794267 after retry RPC after error@1239291227) -> (continuation of Deferred@901023898 after retry RPC after error@1818528854) -> (continuation of Deferred@1904018432 after retry RPC after error@1211151289) -> (continuation of Deferred@1506873981 after retry RPC after error@295641901) -> (continuation of Deferred@140212250 after retry RPC after error@1161543635)))
21:03:55.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.100 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.320 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1970435999(state=PENDING, result=null, callback=(continuation of Deferred@811075722 after retry RPC after error@1220338236) -> (continuation of Deferred@440357120 after retry RPC after error@2090275671) -> (continuation of Deferred@217798679 after retry RPC after error@595195574) -> (continuation of Deferred@1127093384 after retry RPC after error@132622865) -> (continuation of Deferred@1359638810 after retry RPC after error@1026220491) -> (continuation of Deferred@236140722 after retry RPC after error@882818115) -> (continuation of Deferred@916359531 after retry RPC after error@1557140170) -> (continuation of Deferred@1259431754 after retry RPC after error@170702543), errback=(continuation of Deferred@811075722 after retry RPC after error@1220338236) -> (continuation of Deferred@440357120 after retry RPC after error@2090275671) -> (continuation of Deferred@217798679 after retry RPC after error@595195574) -> (continuation of Deferred@1127093384 after retry RPC after error@132622865) -> (continuation of Deferred@1359638810 after retry RPC after error@1026220491) -> (continuation of Deferred@236140722 after retry RPC after error@882818115) -> (continuation of Deferred@916359531 after retry RPC after error@1557140170) -> (continuation of Deferred@1259431754 after retry RPC after error@170702543)))
21:03:55.321 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1970435999(state=PENDING, result=null, callback=(continuation of Deferred@811075722 after retry RPC after error@1220338236) -> (continuation of Deferred@440357120 after retry RPC after error@2090275671) -> (continuation of Deferred@217798679 after retry RPC after error@595195574) -> (continuation of Deferred@1127093384 after retry RPC after error@132622865) -> (continuation of Deferred@1359638810 after retry RPC after error@1026220491) -> (continuation of Deferred@236140722 after retry RPC after error@882818115) -> (continuation of Deferred@916359531 after retry RPC after error@1557140170) -> (continuation of Deferred@1259431754 after retry RPC after error@170702543), errback=(continuation of Deferred@811075722 after retry RPC after error@1220338236) -> (continuation of Deferred@440357120 after retry RPC after error@2090275671) -> (continuation of Deferred@217798679 after retry RPC after error@595195574) -> (continuation of Deferred@1127093384 after retry RPC after error@132622865) -> (continuation of Deferred@1359638810 after retry RPC after error@1026220491) -> (continuation of Deferred@236140722 after retry RPC after error@882818115) -> (continuation of Deferred@916359531 after retry RPC after error@1557140170) -> (continuation of Deferred@1259431754 after retry RPC after error@170702543)))
21:03:55.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.377 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.378 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.579 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1244610592(state=PENDING, result=null, callback=(continuation of Deferred@740537109 after retry RPC after error@425040325) -> (continuation of Deferred@1200402111 after retry RPC after error@2006112302) -> (continuation of Deferred@1176004306 after retry RPC after error@1941328997) -> (continuation of Deferred@1374545402 after retry RPC after error@819926081) -> (continuation of Deferred@905235070 after retry RPC after error@1956685273) -> (continuation of Deferred@1247566978 after retry RPC after error@48837780) -> (continuation of Deferred@299743629 after retry RPC after error@1612863490) -> (continuation of Deferred@1211277512 after retry RPC after error@62152910), errback=(continuation of Deferred@740537109 after retry RPC after error@425040325) -> (continuation of Deferred@1200402111 after retry RPC after error@2006112302) -> (continuation of Deferred@1176004306 after retry RPC after error@1941328997) -> (continuation of Deferred@1374545402 after retry RPC after error@819926081) -> (continuation of Deferred@905235070 after retry RPC after error@1956685273) -> (continuation of Deferred@1247566978 after retry RPC after error@48837780) -> (continuation of Deferred@299743629 after retry RPC after error@1612863490) -> (continuation of Deferred@1211277512 after retry RPC after error@62152910)))
21:03:55.580 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1244610592(state=PENDING, result=null, callback=(continuation of Deferred@740537109 after retry RPC after error@425040325) -> (continuation of Deferred@1200402111 after retry RPC after error@2006112302) -> (continuation of Deferred@1176004306 after retry RPC after error@1941328997) -> (continuation of Deferred@1374545402 after retry RPC after error@819926081) -> (continuation of Deferred@905235070 after retry RPC after error@1956685273) -> (continuation of Deferred@1247566978 after retry RPC after error@48837780) -> (continuation of Deferred@299743629 after retry RPC after error@1612863490) -> (continuation of Deferred@1211277512 after retry RPC after error@62152910), errback=(continuation of Deferred@740537109 after retry RPC after error@425040325) -> (continuation of Deferred@1200402111 after retry RPC after error@2006112302) -> (continuation of Deferred@1176004306 after retry RPC after error@1941328997) -> (continuation of Deferred@1374545402 after retry RPC after error@819926081) -> (continuation of Deferred@905235070 after retry RPC after error@1956685273) -> (continuation of Deferred@1247566978 after retry RPC after error@48837780) -> (continuation of Deferred@299743629 after retry RPC after error@1612863490) -> (continuation of Deferred@1211277512 after retry RPC after error@62152910)))
21:03:55.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.638 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.659 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.799 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2045453139(state=PENDING, result=null, callback=(continuation of Deferred@68544707 after retry RPC after error@344581769) -> (continuation of Deferred@1689041383 after retry RPC after error@1149104026) -> (continuation of Deferred@1862620198 after retry RPC after error@887315169) -> (continuation of Deferred@889302555 after retry RPC after error@506172958) -> (continuation of Deferred@1531598233 after retry RPC after error@1976576067) -> (continuation of Deferred@1301881236 after retry RPC after error@255907167) -> (continuation of Deferred@103788467 after retry RPC after error@1917394181), errback=(continuation of Deferred@68544707 after retry RPC after error@344581769) -> (continuation of Deferred@1689041383 after retry RPC after error@1149104026) -> (continuation of Deferred@1862620198 after retry RPC after error@887315169) -> (continuation of Deferred@889302555 after retry RPC after error@506172958) -> (continuation of Deferred@1531598233 after retry RPC after error@1976576067) -> (continuation of Deferred@1301881236 after retry RPC after error@255907167) -> (continuation of Deferred@103788467 after retry RPC after error@1917394181)))
21:03:55.799 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2045453139(state=PENDING, result=null, callback=(continuation of Deferred@68544707 after retry RPC after error@344581769) -> (continuation of Deferred@1689041383 after retry RPC after error@1149104026) -> (continuation of Deferred@1862620198 after retry RPC after error@887315169) -> (continuation of Deferred@889302555 after retry RPC after error@506172958) -> (continuation of Deferred@1531598233 after retry RPC after error@1976576067) -> (continuation of Deferred@1301881236 after retry RPC after error@255907167) -> (continuation of Deferred@103788467 after retry RPC after error@1917394181), errback=(continuation of Deferred@68544707 after retry RPC after error@344581769) -> (continuation of Deferred@1689041383 after retry RPC after error@1149104026) -> (continuation of Deferred@1862620198 after retry RPC after error@887315169) -> (continuation of Deferred@889302555 after retry RPC after error@506172958) -> (continuation of Deferred@1531598233 after retry RPC after error@1976576067) -> (continuation of Deferred@1301881236 after retry RPC after error@255907167) -> (continuation of Deferred@103788467 after retry RPC after error@1917394181)))
21:03:55.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.857 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.857 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.858 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:55.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:55.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:55.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:55.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:55.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.078 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@126275904(state=PENDING, result=null, callback=(continuation of Deferred@2093314181 after retry RPC after error@58625659) -> (continuation of Deferred@804657352 after retry RPC after error@516673703) -> (continuation of Deferred@1239276195 after retry RPC after error@1042782389) -> (continuation of Deferred@554235873 after retry RPC after error@1636193679) -> (continuation of Deferred@1185514462 after retry RPC after error@1145653646) -> (continuation of Deferred@580393241 after retry RPC after error@1902185192) -> (continuation of Deferred@2069676137 after retry RPC after error@1977082805), errback=(continuation of Deferred@2093314181 after retry RPC after error@58625659) -> (continuation of Deferred@804657352 after retry RPC after error@516673703) -> (continuation of Deferred@1239276195 after retry RPC after error@1042782389) -> (continuation of Deferred@554235873 after retry RPC after error@1636193679) -> (continuation of Deferred@1185514462 after retry RPC after error@1145653646) -> (continuation of Deferred@580393241 after retry RPC after error@1902185192) -> (continuation of Deferred@2069676137 after retry RPC after error@1977082805)))
21:03:56.078 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:56.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.357 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@151708431(state=PENDING, result=null, callback=(continuation of Deferred@501376452 after retry RPC after error@1789227915) -> (continuation of Deferred@1671496686 after retry RPC after error@1030592014) -> (continuation of Deferred@858724386 after retry RPC after error@1926700263) -> (continuation of Deferred@428007200 after retry RPC after error@1208940536) -> (continuation of Deferred@53151955 after retry RPC after error@1238441077) -> (continuation of Deferred@1176871344 after retry RPC after error@2032873403) -> (continuation of Deferred@572378589 after retry RPC after error@1668177759), errback=(continuation of Deferred@501376452 after retry RPC after error@1789227915) -> (continuation of Deferred@1671496686 after retry RPC after error@1030592014) -> (continuation of Deferred@858724386 after retry RPC after error@1926700263) -> (continuation of Deferred@428007200 after retry RPC after error@1208940536) -> (continuation of Deferred@53151955 after retry RPC after error@1238441077) -> (continuation of Deferred@1176871344 after retry RPC after error@2032873403) -> (continuation of Deferred@572378589 after retry RPC after error@1668177759)))
21:03:56.358 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:56.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.417 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.417 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.418 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.520 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2108150973(state=PENDING, result=null, callback=(continuation of Deferred@756531725 after retry RPC after error@308733032) -> (continuation of Deferred@1632247466 after retry RPC after error@579550210) -> (continuation of Deferred@832542196 after retry RPC after error@352285195) -> (continuation of Deferred@636584304 after retry RPC after error@1591857888) -> (continuation of Deferred@320654589 after retry RPC after error@2090634175) -> (continuation of Deferred@1431269213 after retry RPC after error@99209670), errback=(continuation of Deferred@756531725 after retry RPC after error@308733032) -> (continuation of Deferred@1632247466 after retry RPC after error@579550210) -> (continuation of Deferred@832542196 after retry RPC after error@352285195) -> (continuation of Deferred@636584304 after retry RPC after error@1591857888) -> (continuation of Deferred@320654589 after retry RPC after error@2090634175) -> (continuation of Deferred@1431269213 after retry RPC after error@99209670)))
21:03:56.520 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2108150973(state=PENDING, result=null, callback=(continuation of Deferred@756531725 after retry RPC after error@308733032) -> (continuation of Deferred@1632247466 after retry RPC after error@579550210) -> (continuation of Deferred@832542196 after retry RPC after error@352285195) -> (continuation of Deferred@636584304 after retry RPC after error@1591857888) -> (continuation of Deferred@320654589 after retry RPC after error@2090634175) -> (continuation of Deferred@1431269213 after retry RPC after error@99209670), errback=(continuation of Deferred@756531725 after retry RPC after error@308733032) -> (continuation of Deferred@1632247466 after retry RPC after error@579550210) -> (continuation of Deferred@832542196 after retry RPC after error@352285195) -> (continuation of Deferred@636584304 after retry RPC after error@1591857888) -> (continuation of Deferred@320654589 after retry RPC after error@2090634175) -> (continuation of Deferred@1431269213 after retry RPC after error@99209670)))
21:03:56.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.577 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.577 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.577 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.578 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.619 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.637 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.637 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.699 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1570500060(state=PENDING, result=null, callback=(continuation of Deferred@1421232304 after retry RPC after error@1923491713) -> (continuation of Deferred@197687103 after retry RPC after error@155160215) -> (continuation of Deferred@1873458376 after retry RPC after error@754125971) -> (continuation of Deferred@1691019197 after retry RPC after error@292849285) -> (continuation of Deferred@725777559 after retry RPC after error@534870776) -> (continuation of Deferred@749044150 after retry RPC after error@424704521), errback=(continuation of Deferred@1421232304 after retry RPC after error@1923491713) -> (continuation of Deferred@197687103 after retry RPC after error@155160215) -> (continuation of Deferred@1873458376 after retry RPC after error@754125971) -> (continuation of Deferred@1691019197 after retry RPC after error@292849285) -> (continuation of Deferred@725777559 after retry RPC after error@534870776) -> (continuation of Deferred@749044150 after retry RPC after error@424704521)))
21:03:56.700 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1570500060(state=PENDING, result=null, callback=(continuation of Deferred@1421232304 after retry RPC after error@1923491713) -> (continuation of Deferred@197687103 after retry RPC after error@155160215) -> (continuation of Deferred@1873458376 after retry RPC after error@754125971) -> (continuation of Deferred@1691019197 after retry RPC after error@292849285) -> (continuation of Deferred@725777559 after retry RPC after error@534870776) -> (continuation of Deferred@749044150 after retry RPC after error@424704521), errback=(continuation of Deferred@1421232304 after retry RPC after error@1923491713) -> (continuation of Deferred@197687103 after retry RPC after error@155160215) -> (continuation of Deferred@1873458376 after retry RPC after error@754125971) -> (continuation of Deferred@1691019197 after retry RPC after error@292849285) -> (continuation of Deferred@725777559 after retry RPC after error@534870776) -> (continuation of Deferred@749044150 after retry RPC after error@424704521)))
21:03:56.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.757 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.757 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.757 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.758 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.919 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1025370081(state=PENDING, result=null, callback=(continuation of Deferred@946866451 after retry RPC after error@1522355520) -> (continuation of Deferred@2013244908 after retry RPC after error@630156829) -> (continuation of Deferred@495960878 after retry RPC after error@770202258) -> (continuation of Deferred@670175758 after retry RPC after error@2035511147) -> (continuation of Deferred@447104476 after retry RPC after error@1970647248) -> (continuation of Deferred@1117826578 after retry RPC after error@543865114), errback=(continuation of Deferred@946866451 after retry RPC after error@1522355520) -> (continuation of Deferred@2013244908 after retry RPC after error@630156829) -> (continuation of Deferred@495960878 after retry RPC after error@770202258) -> (continuation of Deferred@670175758 after retry RPC after error@2035511147) -> (continuation of Deferred@447104476 after retry RPC after error@1970647248) -> (continuation of Deferred@1117826578 after retry RPC after error@543865114)))
21:03:56.920 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1025370081(state=PENDING, result=null, callback=(continuation of Deferred@946866451 after retry RPC after error@1522355520) -> (continuation of Deferred@2013244908 after retry RPC after error@630156829) -> (continuation of Deferred@495960878 after retry RPC after error@770202258) -> (continuation of Deferred@670175758 after retry RPC after error@2035511147) -> (continuation of Deferred@447104476 after retry RPC after error@1970647248) -> (continuation of Deferred@1117826578 after retry RPC after error@543865114), errback=(continuation of Deferred@946866451 after retry RPC after error@1522355520) -> (continuation of Deferred@2013244908 after retry RPC after error@630156829) -> (continuation of Deferred@495960878 after retry RPC after error@770202258) -> (continuation of Deferred@670175758 after retry RPC after error@2035511147) -> (continuation of Deferred@447104476 after retry RPC after error@1970647248) -> (continuation of Deferred@1117826578 after retry RPC after error@543865114)))
21:03:56.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:56.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:56.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:56.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:56.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:56.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.140 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2013175393(state=PENDING, result=null, callback=(continuation of Deferred@235390610 after retry RPC after error@500397512) -> (continuation of Deferred@1236239298 after retry RPC after error@1243676928) -> (continuation of Deferred@2057465216 after retry RPC after error@2086222377) -> (continuation of Deferred@1230837747 after retry RPC after error@188098265) -> (continuation of Deferred@1645319250 after retry RPC after error@1462526539) -> (continuation of Deferred@1995380178 after retry RPC after error@1289086264) -> (continuation of Deferred@579357684 after retry RPC after error@369249236), errback=(continuation of Deferred@235390610 after retry RPC after error@500397512) -> (continuation of Deferred@1236239298 after retry RPC after error@1243676928) -> (continuation of Deferred@2057465216 after retry RPC after error@2086222377) -> (continuation of Deferred@1230837747 after retry RPC after error@188098265) -> (continuation of Deferred@1645319250 after retry RPC after error@1462526539) -> (continuation of Deferred@1995380178 after retry RPC after error@1289086264) -> (continuation of Deferred@579357684 after retry RPC after error@369249236)))
21:03:57.140 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2013175393(state=PENDING, result=null, callback=(continuation of Deferred@235390610 after retry RPC after error@500397512) -> (continuation of Deferred@1236239298 after retry RPC after error@1243676928) -> (continuation of Deferred@2057465216 after retry RPC after error@2086222377) -> (continuation of Deferred@1230837747 after retry RPC after error@188098265) -> (continuation of Deferred@1645319250 after retry RPC after error@1462526539) -> (continuation of Deferred@1995380178 after retry RPC after error@1289086264) -> (continuation of Deferred@579357684 after retry RPC after error@369249236), errback=(continuation of Deferred@235390610 after retry RPC after error@500397512) -> (continuation of Deferred@1236239298 after retry RPC after error@1243676928) -> (continuation of Deferred@2057465216 after retry RPC after error@2086222377) -> (continuation of Deferred@1230837747 after retry RPC after error@188098265) -> (continuation of Deferred@1645319250 after retry RPC after error@1462526539) -> (continuation of Deferred@1995380178 after retry RPC after error@1289086264) -> (continuation of Deferred@579357684 after retry RPC after error@369249236)))
21:03:57.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.199 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.199 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.418 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@127081199(state=PENDING, result=null, callback=(continuation of Deferred@2023755524 after retry RPC after error@1757570892) -> (continuation of Deferred@1609191080 after retry RPC after error@1085367256) -> (continuation of Deferred@949066981 after retry RPC after error@1125283004) -> (continuation of Deferred@1317076344 after retry RPC after error@98887891) -> (continuation of Deferred@832973374 after retry RPC after error@1631653666) -> (continuation of Deferred@685987151 after retry RPC after error@174297840) -> (continuation of Deferred@177516120 after retry RPC after error@1673893943), errback=(continuation of Deferred@2023755524 after retry RPC after error@1757570892) -> (continuation of Deferred@1609191080 after retry RPC after error@1085367256) -> (continuation of Deferred@949066981 after retry RPC after error@1125283004) -> (continuation of Deferred@1317076344 after retry RPC after error@98887891) -> (continuation of Deferred@832973374 after retry RPC after error@1631653666) -> (continuation of Deferred@685987151 after retry RPC after error@174297840) -> (continuation of Deferred@177516120 after retry RPC after error@1673893943)))
21:03:57.418 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:57.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.497 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.498 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.698 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1285865269(state=PENDING, result=null, callback=(continuation of Deferred@718422315 after retry RPC after error@2131213095) -> (continuation of Deferred@1698690976 after retry RPC after error@210481445) -> (continuation of Deferred@887644806 after retry RPC after error@1995717055) -> (continuation of Deferred@1827426870 after retry RPC after error@1027589143) -> (continuation of Deferred@1253342564 after retry RPC after error@500866124) -> (continuation of Deferred@329846819 after retry RPC after error@1358258493) -> (continuation of Deferred@969111834 after retry RPC after error@1890757266), errback=(continuation of Deferred@718422315 after retry RPC after error@2131213095) -> (continuation of Deferred@1698690976 after retry RPC after error@210481445) -> (continuation of Deferred@887644806 after retry RPC after error@1995717055) -> (continuation of Deferred@1827426870 after retry RPC after error@1027589143) -> (continuation of Deferred@1253342564 after retry RPC after error@500866124) -> (continuation of Deferred@329846819 after retry RPC after error@1358258493) -> (continuation of Deferred@969111834 after retry RPC after error@1890757266)))
21:03:57.698 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:57.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:57.794896 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:57.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:57.799330 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:03:57.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:03:57.800689 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:03:57.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.959 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2036978379(state=PENDING, result=null, callback=(continuation of Deferred@200297072 after retry RPC after error@1057583710) -> (continuation of Deferred@240879055 after retry RPC after error@965996869) -> (continuation of Deferred@817471437 after retry RPC after error@1146523654) -> (continuation of Deferred@384893118 after retry RPC after error@810468092) -> (continuation of Deferred@1282761935 after retry RPC after error@1020464103) -> (continuation of Deferred@582555699 after retry RPC after error@1680238522) -> (continuation of Deferred@51524845 after retry RPC after error@666540891), errback=(continuation of Deferred@200297072 after retry RPC after error@1057583710) -> (continuation of Deferred@240879055 after retry RPC after error@965996869) -> (continuation of Deferred@817471437 after retry RPC after error@1146523654) -> (continuation of Deferred@384893118 after retry RPC after error@810468092) -> (continuation of Deferred@1282761935 after retry RPC after error@1020464103) -> (continuation of Deferred@582555699 after retry RPC after error@1680238522) -> (continuation of Deferred@51524845 after retry RPC after error@666540891)))
21:03:57.960 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2036978379(state=PENDING, result=null, callback=(continuation of Deferred@200297072 after retry RPC after error@1057583710) -> (continuation of Deferred@240879055 after retry RPC after error@965996869) -> (continuation of Deferred@817471437 after retry RPC after error@1146523654) -> (continuation of Deferred@384893118 after retry RPC after error@810468092) -> (continuation of Deferred@1282761935 after retry RPC after error@1020464103) -> (continuation of Deferred@582555699 after retry RPC after error@1680238522) -> (continuation of Deferred@51524845 after retry RPC after error@666540891), errback=(continuation of Deferred@200297072 after retry RPC after error@1057583710) -> (continuation of Deferred@240879055 after retry RPC after error@965996869) -> (continuation of Deferred@817471437 after retry RPC after error@1146523654) -> (continuation of Deferred@384893118 after retry RPC after error@810468092) -> (continuation of Deferred@1282761935 after retry RPC after error@1020464103) -> (continuation of Deferred@582555699 after retry RPC after error@1680238522) -> (continuation of Deferred@51524845 after retry RPC after error@666540891)))
21:03:57.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.977 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.977 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.978 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:57.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:57.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:57.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:57.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:57.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.020 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.079 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.160 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2132207939(state=PENDING, result=null, callback=(continuation of Deferred@1873459794 after retry RPC after error@2076931286) -> (continuation of Deferred@874866726 after retry RPC after error@2083635984) -> (continuation of Deferred@1181982446 after retry RPC after error@1715355442) -> (continuation of Deferred@2034569099 after retry RPC after error@194931440) -> (continuation of Deferred@1371541891 after retry RPC after error@1183871782) -> (continuation of Deferred@402130216 after retry RPC after error@1867068621) -> (continuation of Deferred@1045720238 after retry RPC after error@2065182977) -> (continuation of Deferred@748599823 after retry RPC after error@181869581), errback=(continuation of Deferred@1873459794 after retry RPC after error@2076931286) -> (continuation of Deferred@874866726 after retry RPC after error@2083635984) -> (continuation of Deferred@1181982446 after retry RPC after error@1715355442) -> (continuation of Deferred@2034569099 after retry RPC after error@194931440) -> (continuation of Deferred@1371541891 after retry RPC after error@1183871782) -> (continuation of Deferred@402130216 after retry RPC after error@1867068621) -> (continuation of Deferred@1045720238 after retry RPC after error@2065182977) -> (continuation of Deferred@748599823 after retry RPC after error@181869581)))
21:03:58.160 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2132207939(state=PENDING, result=null, callback=(continuation of Deferred@1873459794 after retry RPC after error@2076931286) -> (continuation of Deferred@874866726 after retry RPC after error@2083635984) -> (continuation of Deferred@1181982446 after retry RPC after error@1715355442) -> (continuation of Deferred@2034569099 after retry RPC after error@194931440) -> (continuation of Deferred@1371541891 after retry RPC after error@1183871782) -> (continuation of Deferred@402130216 after retry RPC after error@1867068621) -> (continuation of Deferred@1045720238 after retry RPC after error@2065182977) -> (continuation of Deferred@748599823 after retry RPC after error@181869581), errback=(continuation of Deferred@1873459794 after retry RPC after error@2076931286) -> (continuation of Deferred@874866726 after retry RPC after error@2083635984) -> (continuation of Deferred@1181982446 after retry RPC after error@1715355442) -> (continuation of Deferred@2034569099 after retry RPC after error@194931440) -> (continuation of Deferred@1371541891 after retry RPC after error@1183871782) -> (continuation of Deferred@402130216 after retry RPC after error@1867068621) -> (continuation of Deferred@1045720238 after retry RPC after error@2065182977) -> (continuation of Deferred@748599823 after retry RPC after error@181869581)))
21:03:58.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.220 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.400 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@603506704(state=PENDING, result=null, callback=(continuation of Deferred@1344803350 after retry RPC after error@922194809) -> (continuation of Deferred@1383124611 after retry RPC after error@925709461) -> (continuation of Deferred@663042244 after retry RPC after error@769858326) -> (continuation of Deferred@2033442047 after retry RPC after error@660915920) -> (continuation of Deferred@698976736 after retry RPC after error@472325224) -> (continuation of Deferred@2064973623 after retry RPC after error@2059920579) -> (continuation of Deferred@524588274 after retry RPC after error@992223022) -> (continuation of Deferred@1729475130 after retry RPC after error@573682663), errback=(continuation of Deferred@1344803350 after retry RPC after error@922194809) -> (continuation of Deferred@1383124611 after retry RPC after error@925709461) -> (continuation of Deferred@663042244 after retry RPC after error@769858326) -> (continuation of Deferred@2033442047 after retry RPC after error@660915920) -> (continuation of Deferred@698976736 after retry RPC after error@472325224) -> (continuation of Deferred@2064973623 after retry RPC after error@2059920579) -> (continuation of Deferred@524588274 after retry RPC after error@992223022) -> (continuation of Deferred@1729475130 after retry RPC after error@573682663)))
21:03:58.400 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@603506704(state=PENDING, result=null, callback=(continuation of Deferred@1344803350 after retry RPC after error@922194809) -> (continuation of Deferred@1383124611 after retry RPC after error@925709461) -> (continuation of Deferred@663042244 after retry RPC after error@769858326) -> (continuation of Deferred@2033442047 after retry RPC after error@660915920) -> (continuation of Deferred@698976736 after retry RPC after error@472325224) -> (continuation of Deferred@2064973623 after retry RPC after error@2059920579) -> (continuation of Deferred@524588274 after retry RPC after error@992223022) -> (continuation of Deferred@1729475130 after retry RPC after error@573682663), errback=(continuation of Deferred@1344803350 after retry RPC after error@922194809) -> (continuation of Deferred@1383124611 after retry RPC after error@925709461) -> (continuation of Deferred@663042244 after retry RPC after error@769858326) -> (continuation of Deferred@2033442047 after retry RPC after error@660915920) -> (continuation of Deferred@698976736 after retry RPC after error@472325224) -> (continuation of Deferred@2064973623 after retry RPC after error@2059920579) -> (continuation of Deferred@524588274 after retry RPC after error@992223022) -> (continuation of Deferred@1729475130 after retry RPC after error@573682663)))
21:03:58.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.660 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1061305552(state=PENDING, result=null, callback=(continuation of Deferred@1690596320 after retry RPC after error@85934151) -> (continuation of Deferred@901209678 after retry RPC after error@457003156) -> (continuation of Deferred@1545694048 after retry RPC after error@591599067) -> (continuation of Deferred@738814675 after retry RPC after error@1572943414) -> (continuation of Deferred@174865541 after retry RPC after error@825892318) -> (continuation of Deferred@1451402968 after retry RPC after error@36068024) -> (continuation of Deferred@1774573165 after retry RPC after error@141619814), errback=(continuation of Deferred@1690596320 after retry RPC after error@85934151) -> (continuation of Deferred@901209678 after retry RPC after error@457003156) -> (continuation of Deferred@1545694048 after retry RPC after error@591599067) -> (continuation of Deferred@738814675 after retry RPC after error@1572943414) -> (continuation of Deferred@174865541 after retry RPC after error@825892318) -> (continuation of Deferred@1451402968 after retry RPC after error@36068024) -> (continuation of Deferred@1774573165 after retry RPC after error@141619814)))
21:03:58.660 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1061305552(state=PENDING, result=null, callback=(continuation of Deferred@1690596320 after retry RPC after error@85934151) -> (continuation of Deferred@901209678 after retry RPC after error@457003156) -> (continuation of Deferred@1545694048 after retry RPC after error@591599067) -> (continuation of Deferred@738814675 after retry RPC after error@1572943414) -> (continuation of Deferred@174865541 after retry RPC after error@825892318) -> (continuation of Deferred@1451402968 after retry RPC after error@36068024) -> (continuation of Deferred@1774573165 after retry RPC after error@141619814), errback=(continuation of Deferred@1690596320 after retry RPC after error@85934151) -> (continuation of Deferred@901209678 after retry RPC after error@457003156) -> (continuation of Deferred@1545694048 after retry RPC after error@591599067) -> (continuation of Deferred@738814675 after retry RPC after error@1572943414) -> (continuation of Deferred@174865541 after retry RPC after error@825892318) -> (continuation of Deferred@1451402968 after retry RPC after error@36068024) -> (continuation of Deferred@1774573165 after retry RPC after error@141619814)))
21:03:58.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.817 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.817 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.938 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1861482693(state=PENDING, result=null, callback=(continuation of Deferred@1002984586 after retry RPC after error@1743398902) -> (continuation of Deferred@1788542568 after retry RPC after error@467535406) -> (continuation of Deferred@1657503195 after retry RPC after error@1411916771) -> (continuation of Deferred@1769795248 after retry RPC after error@49834695) -> (continuation of Deferred@996473574 after retry RPC after error@784789842) -> (continuation of Deferred@549552980 after retry RPC after error@1630432217) -> (continuation of Deferred@2105609936 after retry RPC after error@1683242067) -> (continuation of Deferred@1210304686 after retry RPC after error@1522598608), errback=(continuation of Deferred@1002984586 after retry RPC after error@1743398902) -> (continuation of Deferred@1788542568 after retry RPC after error@467535406) -> (continuation of Deferred@1657503195 after retry RPC after error@1411916771) -> (continuation of Deferred@1769795248 after retry RPC after error@49834695) -> (continuation of Deferred@996473574 after retry RPC after error@784789842) -> (continuation of Deferred@549552980 after retry RPC after error@1630432217) -> (continuation of Deferred@2105609936 after retry RPC after error@1683242067) -> (continuation of Deferred@1210304686 after retry RPC after error@1522598608)))
21:03:58.938 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:58.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:58.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:58.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:58.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:58.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:58.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.179 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@191192237(state=PENDING, result=null, callback=(continuation of Deferred@740035710 after retry RPC after error@1768999427) -> (continuation of Deferred@82621024 after retry RPC after error@634193508) -> (continuation of Deferred@1769902431 after retry RPC after error@112291489) -> (continuation of Deferred@1739568037 after retry RPC after error@714897170) -> (continuation of Deferred@938864503 after retry RPC after error@977034062) -> (continuation of Deferred@546628919 after retry RPC after error@1769546448) -> (continuation of Deferred@1911547615 after retry RPC after error@693187975), errback=(continuation of Deferred@740035710 after retry RPC after error@1768999427) -> (continuation of Deferred@82621024 after retry RPC after error@634193508) -> (continuation of Deferred@1769902431 after retry RPC after error@112291489) -> (continuation of Deferred@1739568037 after retry RPC after error@714897170) -> (continuation of Deferred@938864503 after retry RPC after error@977034062) -> (continuation of Deferred@546628919 after retry RPC after error@1769546448) -> (continuation of Deferred@1911547615 after retry RPC after error@693187975)))
21:03:59.180 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@191192237(state=PENDING, result=null, callback=(continuation of Deferred@740035710 after retry RPC after error@1768999427) -> (continuation of Deferred@82621024 after retry RPC after error@634193508) -> (continuation of Deferred@1769902431 after retry RPC after error@112291489) -> (continuation of Deferred@1739568037 after retry RPC after error@714897170) -> (continuation of Deferred@938864503 after retry RPC after error@977034062) -> (continuation of Deferred@546628919 after retry RPC after error@1769546448) -> (continuation of Deferred@1911547615 after retry RPC after error@693187975), errback=(continuation of Deferred@740035710 after retry RPC after error@1768999427) -> (continuation of Deferred@82621024 after retry RPC after error@634193508) -> (continuation of Deferred@1769902431 after retry RPC after error@112291489) -> (continuation of Deferred@1739568037 after retry RPC after error@714897170) -> (continuation of Deferred@938864503 after retry RPC after error@977034062) -> (continuation of Deferred@546628919 after retry RPC after error@1769546448) -> (continuation of Deferred@1911547615 after retry RPC after error@693187975)))
21:03:59.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.380 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@954294898(state=PENDING, result=null, callback=(continuation of Deferred@234321847 after retry RPC after error@1532954883) -> (continuation of Deferred@835173394 after retry RPC after error@42379946) -> (continuation of Deferred@888811394 after retry RPC after error@390999823) -> (continuation of Deferred@468006616 after retry RPC after error@738270452) -> (continuation of Deferred@1231200457 after retry RPC after error@800250045) -> (continuation of Deferred@500398181 after retry RPC after error@957424023) -> (continuation of Deferred@1705618068 after retry RPC after error@1550315948), errback=(continuation of Deferred@234321847 after retry RPC after error@1532954883) -> (continuation of Deferred@835173394 after retry RPC after error@42379946) -> (continuation of Deferred@888811394 after retry RPC after error@390999823) -> (continuation of Deferred@468006616 after retry RPC after error@738270452) -> (continuation of Deferred@1231200457 after retry RPC after error@800250045) -> (continuation of Deferred@500398181 after retry RPC after error@957424023) -> (continuation of Deferred@1705618068 after retry RPC after error@1550315948)))
21:03:59.380 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@954294898(state=PENDING, result=null, callback=(continuation of Deferred@234321847 after retry RPC after error@1532954883) -> (continuation of Deferred@835173394 after retry RPC after error@42379946) -> (continuation of Deferred@888811394 after retry RPC after error@390999823) -> (continuation of Deferred@468006616 after retry RPC after error@738270452) -> (continuation of Deferred@1231200457 after retry RPC after error@800250045) -> (continuation of Deferred@500398181 after retry RPC after error@957424023) -> (continuation of Deferred@1705618068 after retry RPC after error@1550315948), errback=(continuation of Deferred@234321847 after retry RPC after error@1532954883) -> (continuation of Deferred@835173394 after retry RPC after error@42379946) -> (continuation of Deferred@888811394 after retry RPC after error@390999823) -> (continuation of Deferred@468006616 after retry RPC after error@738270452) -> (continuation of Deferred@1231200457 after retry RPC after error@800250045) -> (continuation of Deferred@500398181 after retry RPC after error@957424023) -> (continuation of Deferred@1705618068 after retry RPC after error@1550315948)))
21:03:59.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.420 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.420 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.420 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.420 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.658 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@520692904(state=PENDING, result=null, callback=(continuation of Deferred@826798569 after retry RPC after error@720155495) -> (continuation of Deferred@155179557 after retry RPC after error@1606783950) -> (continuation of Deferred@1359214950 after retry RPC after error@507191210) -> (continuation of Deferred@1763840186 after retry RPC after error@445294003) -> (continuation of Deferred@1418508109 after retry RPC after error@302789689) -> (continuation of Deferred@1808945410 after retry RPC after error@616220406) -> (continuation of Deferred@364723187 after retry RPC after error@1717794945), errback=(continuation of Deferred@826798569 after retry RPC after error@720155495) -> (continuation of Deferred@155179557 after retry RPC after error@1606783950) -> (continuation of Deferred@1359214950 after retry RPC after error@507191210) -> (continuation of Deferred@1763840186 after retry RPC after error@445294003) -> (continuation of Deferred@1418508109 after retry RPC after error@302789689) -> (continuation of Deferred@1808945410 after retry RPC after error@616220406) -> (continuation of Deferred@364723187 after retry RPC after error@1717794945)))
21:03:59.658 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:03:59.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.680 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.680 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.860 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@540379364(state=PENDING, result=null, callback=(continuation of Deferred@1861427595 after retry RPC after error@278020876) -> (continuation of Deferred@1283289194 after retry RPC after error@1073984425) -> (continuation of Deferred@931867995 after retry RPC after error@897815173) -> (continuation of Deferred@307907199 after retry RPC after error@1327050086) -> (continuation of Deferred@555098155 after retry RPC after error@136874294) -> (continuation of Deferred@1222718462 after retry RPC after error@1318405529) -> (continuation of Deferred@1578251032 after retry RPC after error@306143445), errback=(continuation of Deferred@1861427595 after retry RPC after error@278020876) -> (continuation of Deferred@1283289194 after retry RPC after error@1073984425) -> (continuation of Deferred@931867995 after retry RPC after error@897815173) -> (continuation of Deferred@307907199 after retry RPC after error@1327050086) -> (continuation of Deferred@555098155 after retry RPC after error@136874294) -> (continuation of Deferred@1222718462 after retry RPC after error@1318405529) -> (continuation of Deferred@1578251032 after retry RPC after error@306143445)))
21:03:59.860 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@540379364(state=PENDING, result=null, callback=(continuation of Deferred@1861427595 after retry RPC after error@278020876) -> (continuation of Deferred@1283289194 after retry RPC after error@1073984425) -> (continuation of Deferred@931867995 after retry RPC after error@897815173) -> (continuation of Deferred@307907199 after retry RPC after error@1327050086) -> (continuation of Deferred@555098155 after retry RPC after error@136874294) -> (continuation of Deferred@1222718462 after retry RPC after error@1318405529) -> (continuation of Deferred@1578251032 after retry RPC after error@306143445), errback=(continuation of Deferred@1861427595 after retry RPC after error@278020876) -> (continuation of Deferred@1283289194 after retry RPC after error@1073984425) -> (continuation of Deferred@931867995 after retry RPC after error@897815173) -> (continuation of Deferred@307907199 after retry RPC after error@1327050086) -> (continuation of Deferred@555098155 after retry RPC after error@136874294) -> (continuation of Deferred@1222718462 after retry RPC after error@1318405529) -> (continuation of Deferred@1578251032 after retry RPC after error@306143445)))
21:03:59.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:03:59.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:03:59.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:03:59.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:03:59.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:03:59.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.080 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@33854213(state=PENDING, result=null, callback=(continuation of Deferred@586897052 after retry RPC after error@669260312) -> (continuation of Deferred@1224708092 after retry RPC after error@1850057053) -> (continuation of Deferred@376591807 after retry RPC after error@289520051) -> (continuation of Deferred@628446216 after retry RPC after error@1719223986) -> (continuation of Deferred@1646405743 after retry RPC after error@2125629881) -> (continuation of Deferred@1902659443 after retry RPC after error@1560859450) -> (continuation of Deferred@773515684 after retry RPC after error@1141854879), errback=(continuation of Deferred@586897052 after retry RPC after error@669260312) -> (continuation of Deferred@1224708092 after retry RPC after error@1850057053) -> (continuation of Deferred@376591807 after retry RPC after error@289520051) -> (continuation of Deferred@628446216 after retry RPC after error@1719223986) -> (continuation of Deferred@1646405743 after retry RPC after error@2125629881) -> (continuation of Deferred@1902659443 after retry RPC after error@1560859450) -> (continuation of Deferred@773515684 after retry RPC after error@1141854879)))
21:04:00.080 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@33854213(state=PENDING, result=null, callback=(continuation of Deferred@586897052 after retry RPC after error@669260312) -> (continuation of Deferred@1224708092 after retry RPC after error@1850057053) -> (continuation of Deferred@376591807 after retry RPC after error@289520051) -> (continuation of Deferred@628446216 after retry RPC after error@1719223986) -> (continuation of Deferred@1646405743 after retry RPC after error@2125629881) -> (continuation of Deferred@1902659443 after retry RPC after error@1560859450) -> (continuation of Deferred@773515684 after retry RPC after error@1141854879), errback=(continuation of Deferred@586897052 after retry RPC after error@669260312) -> (continuation of Deferred@1224708092 after retry RPC after error@1850057053) -> (continuation of Deferred@376591807 after retry RPC after error@289520051) -> (continuation of Deferred@628446216 after retry RPC after error@1719223986) -> (continuation of Deferred@1646405743 after retry RPC after error@2125629881) -> (continuation of Deferred@1902659443 after retry RPC after error@1560859450) -> (continuation of Deferred@773515684 after retry RPC after error@1141854879)))
21:04:00.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.140 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.140 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.140 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.219 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.279 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1615070797(state=PENDING, result=null, callback=(continuation of Deferred@903521692 after retry RPC after error@1257148690) -> (continuation of Deferred@1539353316 after retry RPC after error@469154239) -> (continuation of Deferred@2069891352 after retry RPC after error@1161322855) -> (continuation of Deferred@468808878 after retry RPC after error@895749422) -> (continuation of Deferred@1312291411 after retry RPC after error@1216514475) -> (continuation of Deferred@1481035793 after retry RPC after error@1306442570) -> (continuation of Deferred@1011496384 after retry RPC after error@1624845363), errback=(continuation of Deferred@903521692 after retry RPC after error@1257148690) -> (continuation of Deferred@1539353316 after retry RPC after error@469154239) -> (continuation of Deferred@2069891352 after retry RPC after error@1161322855) -> (continuation of Deferred@468808878 after retry RPC after error@895749422) -> (continuation of Deferred@1312291411 after retry RPC after error@1216514475) -> (continuation of Deferred@1481035793 after retry RPC after error@1306442570) -> (continuation of Deferred@1011496384 after retry RPC after error@1624845363)))
21:04:00.279 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1615070797(state=PENDING, result=null, callback=(continuation of Deferred@903521692 after retry RPC after error@1257148690) -> (continuation of Deferred@1539353316 after retry RPC after error@469154239) -> (continuation of Deferred@2069891352 after retry RPC after error@1161322855) -> (continuation of Deferred@468808878 after retry RPC after error@895749422) -> (continuation of Deferred@1312291411 after retry RPC after error@1216514475) -> (continuation of Deferred@1481035793 after retry RPC after error@1306442570) -> (continuation of Deferred@1011496384 after retry RPC after error@1624845363), errback=(continuation of Deferred@903521692 after retry RPC after error@1257148690) -> (continuation of Deferred@1539353316 after retry RPC after error@469154239) -> (continuation of Deferred@2069891352 after retry RPC after error@1161322855) -> (continuation of Deferred@468808878 after retry RPC after error@895749422) -> (continuation of Deferred@1312291411 after retry RPC after error@1216514475) -> (continuation of Deferred@1481035793 after retry RPC after error@1306442570) -> (continuation of Deferred@1011496384 after retry RPC after error@1624845363)))
21:04:00.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.457 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.459 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1205132761(state=PENDING, result=null, callback=(continuation of Deferred@119030416 after retry RPC after error@1084192061) -> (continuation of Deferred@1455411208 after retry RPC after error@929359123) -> (continuation of Deferred@1915563437 after retry RPC after error@1235257416) -> (continuation of Deferred@1639399140 after retry RPC after error@1911231420) -> (continuation of Deferred@1789921385 after retry RPC after error@664747967) -> (continuation of Deferred@2139579830 after retry RPC after error@1448668937) -> (continuation of Deferred@1019299563 after retry RPC after error@1652421643), errback=(continuation of Deferred@119030416 after retry RPC after error@1084192061) -> (continuation of Deferred@1455411208 after retry RPC after error@929359123) -> (continuation of Deferred@1915563437 after retry RPC after error@1235257416) -> (continuation of Deferred@1639399140 after retry RPC after error@1911231420) -> (continuation of Deferred@1789921385 after retry RPC after error@664747967) -> (continuation of Deferred@2139579830 after retry RPC after error@1448668937) -> (continuation of Deferred@1019299563 after retry RPC after error@1652421643)))
21:04:00.460 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1205132761(state=PENDING, result=null, callback=(continuation of Deferred@119030416 after retry RPC after error@1084192061) -> (continuation of Deferred@1455411208 after retry RPC after error@929359123) -> (continuation of Deferred@1915563437 after retry RPC after error@1235257416) -> (continuation of Deferred@1639399140 after retry RPC after error@1911231420) -> (continuation of Deferred@1789921385 after retry RPC after error@664747967) -> (continuation of Deferred@2139579830 after retry RPC after error@1448668937) -> (continuation of Deferred@1019299563 after retry RPC after error@1652421643), errback=(continuation of Deferred@119030416 after retry RPC after error@1084192061) -> (continuation of Deferred@1455411208 after retry RPC after error@929359123) -> (continuation of Deferred@1915563437 after retry RPC after error@1235257416) -> (continuation of Deferred@1639399140 after retry RPC after error@1911231420) -> (continuation of Deferred@1789921385 after retry RPC after error@664747967) -> (continuation of Deferred@2139579830 after retry RPC after error@1448668937) -> (continuation of Deferred@1019299563 after retry RPC after error@1652421643)))
21:04:00.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.539 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.558 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.660 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1322572686(state=PENDING, result=null, callback=(continuation of Deferred@999591938 after retry RPC after error@906900424) -> (continuation of Deferred@571885247 after retry RPC after error@1257697752) -> (continuation of Deferred@1442671620 after retry RPC after error@313165056) -> (continuation of Deferred@43584084 after retry RPC after error@2132562926) -> (continuation of Deferred@1094683973 after retry RPC after error@125539047) -> (continuation of Deferred@1308292086 after retry RPC after error@1357142660) -> (continuation of Deferred@1837936760 after retry RPC after error@918285188), errback=(continuation of Deferred@999591938 after retry RPC after error@906900424) -> (continuation of Deferred@571885247 after retry RPC after error@1257697752) -> (continuation of Deferred@1442671620 after retry RPC after error@313165056) -> (continuation of Deferred@43584084 after retry RPC after error@2132562926) -> (continuation of Deferred@1094683973 after retry RPC after error@125539047) -> (continuation of Deferred@1308292086 after retry RPC after error@1357142660) -> (continuation of Deferred@1837936760 after retry RPC after error@918285188)))
21:04:00.660 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1322572686(state=PENDING, result=null, callback=(continuation of Deferred@999591938 after retry RPC after error@906900424) -> (continuation of Deferred@571885247 after retry RPC after error@1257697752) -> (continuation of Deferred@1442671620 after retry RPC after error@313165056) -> (continuation of Deferred@43584084 after retry RPC after error@2132562926) -> (continuation of Deferred@1094683973 after retry RPC after error@125539047) -> (continuation of Deferred@1308292086 after retry RPC after error@1357142660) -> (continuation of Deferred@1837936760 after retry RPC after error@918285188), errback=(continuation of Deferred@999591938 after retry RPC after error@906900424) -> (continuation of Deferred@571885247 after retry RPC after error@1257697752) -> (continuation of Deferred@1442671620 after retry RPC after error@313165056) -> (continuation of Deferred@43584084 after retry RPC after error@2132562926) -> (continuation of Deferred@1094683973 after retry RPC after error@125539047) -> (continuation of Deferred@1308292086 after retry RPC after error@1357142660) -> (continuation of Deferred@1837936760 after retry RPC after error@918285188)))
21:04:00.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.699 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.938 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@52839699(state=PENDING, result=null, callback=(continuation of Deferred@21705591 after retry RPC after error@1418731023) -> (continuation of Deferred@1561980784 after retry RPC after error@1814443805) -> (continuation of Deferred@883843429 after retry RPC after error@1612341657) -> (continuation of Deferred@1927966853 after retry RPC after error@791232282) -> (continuation of Deferred@1758189267 after retry RPC after error@1542350074) -> (continuation of Deferred@2057394522 after retry RPC after error@471230715) -> (continuation of Deferred@710760676 after retry RPC after error@45883617) -> (continuation of Deferred@1923596901 after retry RPC after error@1566125434), errback=(continuation of Deferred@21705591 after retry RPC after error@1418731023) -> (continuation of Deferred@1561980784 after retry RPC after error@1814443805) -> (continuation of Deferred@883843429 after retry RPC after error@1612341657) -> (continuation of Deferred@1927966853 after retry RPC after error@791232282) -> (continuation of Deferred@1758189267 after retry RPC after error@1542350074) -> (continuation of Deferred@2057394522 after retry RPC after error@471230715) -> (continuation of Deferred@710760676 after retry RPC after error@45883617) -> (continuation of Deferred@1923596901 after retry RPC after error@1566125434)))
21:04:00.938 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:00.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:00.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:00.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:00.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:00.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:00.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.039 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.160 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1275526197(state=PENDING, result=null, callback=(continuation of Deferred@1241638125 after retry RPC after error@919545930) -> (continuation of Deferred@2119454469 after retry RPC after error@88667054) -> (continuation of Deferred@1084967270 after retry RPC after error@258585665) -> (continuation of Deferred@1402601166 after retry RPC after error@294564496) -> (continuation of Deferred@192162238 after retry RPC after error@1241341741) -> (continuation of Deferred@1435416402 after retry RPC after error@837850433) -> (continuation of Deferred@459502360 after retry RPC after error@952390510), errback=(continuation of Deferred@1241638125 after retry RPC after error@919545930) -> (continuation of Deferred@2119454469 after retry RPC after error@88667054) -> (continuation of Deferred@1084967270 after retry RPC after error@258585665) -> (continuation of Deferred@1402601166 after retry RPC after error@294564496) -> (continuation of Deferred@192162238 after retry RPC after error@1241341741) -> (continuation of Deferred@1435416402 after retry RPC after error@837850433) -> (continuation of Deferred@459502360 after retry RPC after error@952390510)))
21:04:01.160 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1275526197(state=PENDING, result=null, callback=(continuation of Deferred@1241638125 after retry RPC after error@919545930) -> (continuation of Deferred@2119454469 after retry RPC after error@88667054) -> (continuation of Deferred@1084967270 after retry RPC after error@258585665) -> (continuation of Deferred@1402601166 after retry RPC after error@294564496) -> (continuation of Deferred@192162238 after retry RPC after error@1241341741) -> (continuation of Deferred@1435416402 after retry RPC after error@837850433) -> (continuation of Deferred@459502360 after retry RPC after error@952390510), errback=(continuation of Deferred@1241638125 after retry RPC after error@919545930) -> (continuation of Deferred@2119454469 after retry RPC after error@88667054) -> (continuation of Deferred@1084967270 after retry RPC after error@258585665) -> (continuation of Deferred@1402601166 after retry RPC after error@294564496) -> (continuation of Deferred@192162238 after retry RPC after error@1241341741) -> (continuation of Deferred@1435416402 after retry RPC after error@837850433) -> (continuation of Deferred@459502360 after retry RPC after error@952390510)))
21:04:01.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.239 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.419 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@741808103(state=PENDING, result=null, callback=(continuation of Deferred@1575897125 after retry RPC after error@1673068365) -> (continuation of Deferred@533635190 after retry RPC after error@189928519) -> (continuation of Deferred@609405544 after retry RPC after error@11803448) -> (continuation of Deferred@1788073544 after retry RPC after error@1903691633) -> (continuation of Deferred@1137410748 after retry RPC after error@1669737510) -> (continuation of Deferred@391350152 after retry RPC after error@648104348) -> (continuation of Deferred@971039126 after retry RPC after error@1809128848), errback=(continuation of Deferred@1575897125 after retry RPC after error@1673068365) -> (continuation of Deferred@533635190 after retry RPC after error@189928519) -> (continuation of Deferred@609405544 after retry RPC after error@11803448) -> (continuation of Deferred@1788073544 after retry RPC after error@1903691633) -> (continuation of Deferred@1137410748 after retry RPC after error@1669737510) -> (continuation of Deferred@391350152 after retry RPC after error@648104348) -> (continuation of Deferred@971039126 after retry RPC after error@1809128848)))
21:04:01.420 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@741808103(state=PENDING, result=null, callback=(continuation of Deferred@1575897125 after retry RPC after error@1673068365) -> (continuation of Deferred@533635190 after retry RPC after error@189928519) -> (continuation of Deferred@609405544 after retry RPC after error@11803448) -> (continuation of Deferred@1788073544 after retry RPC after error@1903691633) -> (continuation of Deferred@1137410748 after retry RPC after error@1669737510) -> (continuation of Deferred@391350152 after retry RPC after error@648104348) -> (continuation of Deferred@971039126 after retry RPC after error@1809128848), errback=(continuation of Deferred@1575897125 after retry RPC after error@1673068365) -> (continuation of Deferred@533635190 after retry RPC after error@189928519) -> (continuation of Deferred@609405544 after retry RPC after error@11803448) -> (continuation of Deferred@1788073544 after retry RPC after error@1903691633) -> (continuation of Deferred@1137410748 after retry RPC after error@1669737510) -> (continuation of Deferred@391350152 after retry RPC after error@648104348) -> (continuation of Deferred@971039126 after retry RPC after error@1809128848)))
21:04:01.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.439 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.439 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.497 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.497 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.498 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.619 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.620 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@269731426(state=PENDING, result=null, callback=(continuation of Deferred@1295456293 after retry RPC after error@514470715) -> (continuation of Deferred@1116696437 after retry RPC after error@1285652880) -> (continuation of Deferred@958553617 after retry RPC after error@1994297829) -> (continuation of Deferred@1335078028 after retry RPC after error@268311023) -> (continuation of Deferred@612834519 after retry RPC after error@643096603) -> (continuation of Deferred@1962008465 after retry RPC after error@188711297), errback=(continuation of Deferred@1295456293 after retry RPC after error@514470715) -> (continuation of Deferred@1116696437 after retry RPC after error@1285652880) -> (continuation of Deferred@958553617 after retry RPC after error@1994297829) -> (continuation of Deferred@1335078028 after retry RPC after error@268311023) -> (continuation of Deferred@612834519 after retry RPC after error@643096603) -> (continuation of Deferred@1962008465 after retry RPC after error@188711297)))
21:04:01.620 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@269731426(state=PENDING, result=null, callback=(continuation of Deferred@1295456293 after retry RPC after error@514470715) -> (continuation of Deferred@1116696437 after retry RPC after error@1285652880) -> (continuation of Deferred@958553617 after retry RPC after error@1994297829) -> (continuation of Deferred@1335078028 after retry RPC after error@268311023) -> (continuation of Deferred@612834519 after retry RPC after error@643096603) -> (continuation of Deferred@1962008465 after retry RPC after error@188711297), errback=(continuation of Deferred@1295456293 after retry RPC after error@514470715) -> (continuation of Deferred@1116696437 after retry RPC after error@1285652880) -> (continuation of Deferred@958553617 after retry RPC after error@1994297829) -> (continuation of Deferred@1335078028 after retry RPC after error@268311023) -> (continuation of Deferred@612834519 after retry RPC after error@643096603) -> (continuation of Deferred@1962008465 after retry RPC after error@188711297)))
21:04:01.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.679 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.680 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.840 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1344077774(state=PENDING, result=null, callback=(continuation of Deferred@142804652 after retry RPC after error@194617262) -> (continuation of Deferred@562109934 after retry RPC after error@383022239) -> (continuation of Deferred@108181135 after retry RPC after error@1095838976) -> (continuation of Deferred@1746286249 after retry RPC after error@1565775800) -> (continuation of Deferred@1317454931 after retry RPC after error@116902318) -> (continuation of Deferred@1594688679 after retry RPC after error@480306921) -> (continuation of Deferred@1694240114 after retry RPC after error@1551762170), errback=(continuation of Deferred@142804652 after retry RPC after error@194617262) -> (continuation of Deferred@562109934 after retry RPC after error@383022239) -> (continuation of Deferred@108181135 after retry RPC after error@1095838976) -> (continuation of Deferred@1746286249 after retry RPC after error@1565775800) -> (continuation of Deferred@1317454931 after retry RPC after error@116902318) -> (continuation of Deferred@1594688679 after retry RPC after error@480306921) -> (continuation of Deferred@1694240114 after retry RPC after error@1551762170)))
21:04:01.840 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1344077774(state=PENDING, result=null, callback=(continuation of Deferred@142804652 after retry RPC after error@194617262) -> (continuation of Deferred@562109934 after retry RPC after error@383022239) -> (continuation of Deferred@108181135 after retry RPC after error@1095838976) -> (continuation of Deferred@1746286249 after retry RPC after error@1565775800) -> (continuation of Deferred@1317454931 after retry RPC after error@116902318) -> (continuation of Deferred@1594688679 after retry RPC after error@480306921) -> (continuation of Deferred@1694240114 after retry RPC after error@1551762170), errback=(continuation of Deferred@142804652 after retry RPC after error@194617262) -> (continuation of Deferred@562109934 after retry RPC after error@383022239) -> (continuation of Deferred@108181135 after retry RPC after error@1095838976) -> (continuation of Deferred@1746286249 after retry RPC after error@1565775800) -> (continuation of Deferred@1317454931 after retry RPC after error@116902318) -> (continuation of Deferred@1594688679 after retry RPC after error@480306921) -> (continuation of Deferred@1694240114 after retry RPC after error@1551762170)))
21:04:01.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:01.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:01.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:01.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:01.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:01.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.080 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1561607549(state=PENDING, result=null, callback=(continuation of Deferred@1384389336 after retry RPC after error@163373571) -> (continuation of Deferred@1349379520 after retry RPC after error@753845605) -> (continuation of Deferred@1590577402 after retry RPC after error@814487002) -> (continuation of Deferred@267429358 after retry RPC after error@1433377944) -> (continuation of Deferred@2043320151 after retry RPC after error@691564401) -> (continuation of Deferred@602759277 after retry RPC after error@181384148) -> (continuation of Deferred@655556085 after retry RPC after error@1304124801), errback=(continuation of Deferred@1384389336 after retry RPC after error@163373571) -> (continuation of Deferred@1349379520 after retry RPC after error@753845605) -> (continuation of Deferred@1590577402 after retry RPC after error@814487002) -> (continuation of Deferred@267429358 after retry RPC after error@1433377944) -> (continuation of Deferred@2043320151 after retry RPC after error@691564401) -> (continuation of Deferred@602759277 after retry RPC after error@181384148) -> (continuation of Deferred@655556085 after retry RPC after error@1304124801)))
21:04:02.080 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1561607549(state=PENDING, result=null, callback=(continuation of Deferred@1384389336 after retry RPC after error@163373571) -> (continuation of Deferred@1349379520 after retry RPC after error@753845605) -> (continuation of Deferred@1590577402 after retry RPC after error@814487002) -> (continuation of Deferred@267429358 after retry RPC after error@1433377944) -> (continuation of Deferred@2043320151 after retry RPC after error@691564401) -> (continuation of Deferred@602759277 after retry RPC after error@181384148) -> (continuation of Deferred@655556085 after retry RPC after error@1304124801), errback=(continuation of Deferred@1384389336 after retry RPC after error@163373571) -> (continuation of Deferred@1349379520 after retry RPC after error@753845605) -> (continuation of Deferred@1590577402 after retry RPC after error@814487002) -> (continuation of Deferred@267429358 after retry RPC after error@1433377944) -> (continuation of Deferred@2043320151 after retry RPC after error@691564401) -> (continuation of Deferred@602759277 after retry RPC after error@181384148) -> (continuation of Deferred@655556085 after retry RPC after error@1304124801)))
21:04:02.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.100 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.300 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@157783447(state=PENDING, result=null, callback=(continuation of Deferred@1106074060 after retry RPC after error@901168487) -> (continuation of Deferred@1683270086 after retry RPC after error@1370117110) -> (continuation of Deferred@2057458486 after retry RPC after error@2063570048) -> (continuation of Deferred@1000605824 after retry RPC after error@940755425) -> (continuation of Deferred@1354831414 after retry RPC after error@1453114170) -> (continuation of Deferred@1306997089 after retry RPC after error@1662297341) -> (continuation of Deferred@1002204687 after retry RPC after error@185204561) -> (continuation of Deferred@2131864424 after retry RPC after error@962911035), errback=(continuation of Deferred@1106074060 after retry RPC after error@901168487) -> (continuation of Deferred@1683270086 after retry RPC after error@1370117110) -> (continuation of Deferred@2057458486 after retry RPC after error@2063570048) -> (continuation of Deferred@1000605824 after retry RPC after error@940755425) -> (continuation of Deferred@1354831414 after retry RPC after error@1453114170) -> (continuation of Deferred@1306997089 after retry RPC after error@1662297341) -> (continuation of Deferred@1002204687 after retry RPC after error@185204561) -> (continuation of Deferred@2131864424 after retry RPC after error@962911035)))
21:04:02.301 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@157783447(state=PENDING, result=null, callback=(continuation of Deferred@1106074060 after retry RPC after error@901168487) -> (continuation of Deferred@1683270086 after retry RPC after error@1370117110) -> (continuation of Deferred@2057458486 after retry RPC after error@2063570048) -> (continuation of Deferred@1000605824 after retry RPC after error@940755425) -> (continuation of Deferred@1354831414 after retry RPC after error@1453114170) -> (continuation of Deferred@1306997089 after retry RPC after error@1662297341) -> (continuation of Deferred@1002204687 after retry RPC after error@185204561) -> (continuation of Deferred@2131864424 after retry RPC after error@962911035), errback=(continuation of Deferred@1106074060 after retry RPC after error@901168487) -> (continuation of Deferred@1683270086 after retry RPC after error@1370117110) -> (continuation of Deferred@2057458486 after retry RPC after error@2063570048) -> (continuation of Deferred@1000605824 after retry RPC after error@940755425) -> (continuation of Deferred@1354831414 after retry RPC after error@1453114170) -> (continuation of Deferred@1306997089 after retry RPC after error@1662297341) -> (continuation of Deferred@1002204687 after retry RPC after error@185204561) -> (continuation of Deferred@2131864424 after retry RPC after error@962911035)))
21:04:02.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.400 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.400 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.400 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.401 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.403 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.560 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [85ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1846241021(state=PENDING, result=null, callback=(continuation of Deferred@1097088094 after retry RPC after error@1435288262) -> (continuation of Deferred@224029368 after retry RPC after error@953407840) -> (continuation of Deferred@1479902440 after retry RPC after error@1805449178) -> (continuation of Deferred@894259131 after retry RPC after error@326338731) -> (continuation of Deferred@1729453289 after retry RPC after error@1799050742) -> (continuation of Deferred@877985725 after retry RPC after error@1005395788) -> (continuation of Deferred@1993535180 after retry RPC after error@1207406853), errback=(continuation of Deferred@1097088094 after retry RPC after error@1435288262) -> (continuation of Deferred@224029368 after retry RPC after error@953407840) -> (continuation of Deferred@1479902440 after retry RPC after error@1805449178) -> (continuation of Deferred@894259131 after retry RPC after error@326338731) -> (continuation of Deferred@1729453289 after retry RPC after error@1799050742) -> (continuation of Deferred@877985725 after retry RPC after error@1005395788) -> (continuation of Deferred@1993535180 after retry RPC after error@1207406853)))
21:04:02.560 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [85ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1846241021(state=PENDING, result=null, callback=(continuation of Deferred@1097088094 after retry RPC after error@1435288262) -> (continuation of Deferred@224029368 after retry RPC after error@953407840) -> (continuation of Deferred@1479902440 after retry RPC after error@1805449178) -> (continuation of Deferred@894259131 after retry RPC after error@326338731) -> (continuation of Deferred@1729453289 after retry RPC after error@1799050742) -> (continuation of Deferred@877985725 after retry RPC after error@1005395788) -> (continuation of Deferred@1993535180 after retry RPC after error@1207406853), errback=(continuation of Deferred@1097088094 after retry RPC after error@1435288262) -> (continuation of Deferred@224029368 after retry RPC after error@953407840) -> (continuation of Deferred@1479902440 after retry RPC after error@1805449178) -> (continuation of Deferred@894259131 after retry RPC after error@326338731) -> (continuation of Deferred@1729453289 after retry RPC after error@1799050742) -> (continuation of Deferred@877985725 after retry RPC after error@1005395788) -> (continuation of Deferred@1993535180 after retry RPC after error@1207406853)))
21:04:02.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.677 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.677 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.677 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.740 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.740 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1569678401(state=PENDING, result=null, callback=(continuation of Deferred@1383470482 after retry RPC after error@1296836570) -> (continuation of Deferred@1034161038 after retry RPC after error@464506135) -> (continuation of Deferred@2039972213 after retry RPC after error@1439239018) -> (continuation of Deferred@1210876454 after retry RPC after error@755906881) -> (continuation of Deferred@1750571452 after retry RPC after error@1977113991) -> (continuation of Deferred@1506401448 after retry RPC after error@97592033), errback=(continuation of Deferred@1383470482 after retry RPC after error@1296836570) -> (continuation of Deferred@1034161038 after retry RPC after error@464506135) -> (continuation of Deferred@2039972213 after retry RPC after error@1439239018) -> (continuation of Deferred@1210876454 after retry RPC after error@755906881) -> (continuation of Deferred@1750571452 after retry RPC after error@1977113991) -> (continuation of Deferred@1506401448 after retry RPC after error@97592033)))
21:04:02.741 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1569678401(state=PENDING, result=null, callback=(continuation of Deferred@1383470482 after retry RPC after error@1296836570) -> (continuation of Deferred@1034161038 after retry RPC after error@464506135) -> (continuation of Deferred@2039972213 after retry RPC after error@1439239018) -> (continuation of Deferred@1210876454 after retry RPC after error@755906881) -> (continuation of Deferred@1750571452 after retry RPC after error@1977113991) -> (continuation of Deferred@1506401448 after retry RPC after error@97592033), errback=(continuation of Deferred@1383470482 after retry RPC after error@1296836570) -> (continuation of Deferred@1034161038 after retry RPC after error@464506135) -> (continuation of Deferred@2039972213 after retry RPC after error@1439239018) -> (continuation of Deferred@1210876454 after retry RPC after error@755906881) -> (continuation of Deferred@1750571452 after retry RPC after error@1977113991) -> (continuation of Deferred@1506401448 after retry RPC after error@97592033)))
21:04:02.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.779 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.803 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.803 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.803 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.803 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.803 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.803 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.803 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:02.815629 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:04:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:02.817013 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:04:02.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:02.818811 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:04:02.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:02.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:02.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:02.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:02.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:02.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.000 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [46ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@16809227(state=PENDING, result=null, callback=(continuation of Deferred@129152110 after retry RPC after error@1735510121) -> (continuation of Deferred@1701709238 after retry RPC after error@1728365621) -> (continuation of Deferred@1963396479 after retry RPC after error@475832630) -> (continuation of Deferred@151667902 after retry RPC after error@1223564486) -> (continuation of Deferred@89099345 after retry RPC after error@2086918367) -> (continuation of Deferred@664231001 after retry RPC after error@531070017) -> (continuation of Deferred@1718834912 after retry RPC after error@70569937), errback=(continuation of Deferred@129152110 after retry RPC after error@1735510121) -> (continuation of Deferred@1701709238 after retry RPC after error@1728365621) -> (continuation of Deferred@1963396479 after retry RPC after error@475832630) -> (continuation of Deferred@151667902 after retry RPC after error@1223564486) -> (continuation of Deferred@89099345 after retry RPC after error@2086918367) -> (continuation of Deferred@664231001 after retry RPC after error@531070017) -> (continuation of Deferred@1718834912 after retry RPC after error@70569937)))
21:04:03.000 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [46ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@16809227(state=PENDING, result=null, callback=(continuation of Deferred@129152110 after retry RPC after error@1735510121) -> (continuation of Deferred@1701709238 after retry RPC after error@1728365621) -> (continuation of Deferred@1963396479 after retry RPC after error@475832630) -> (continuation of Deferred@151667902 after retry RPC after error@1223564486) -> (continuation of Deferred@89099345 after retry RPC after error@2086918367) -> (continuation of Deferred@664231001 after retry RPC after error@531070017) -> (continuation of Deferred@1718834912 after retry RPC after error@70569937), errback=(continuation of Deferred@129152110 after retry RPC after error@1735510121) -> (continuation of Deferred@1701709238 after retry RPC after error@1728365621) -> (continuation of Deferred@1963396479 after retry RPC after error@475832630) -> (continuation of Deferred@151667902 after retry RPC after error@1223564486) -> (continuation of Deferred@89099345 after retry RPC after error@2086918367) -> (continuation of Deferred@664231001 after retry RPC after error@531070017) -> (continuation of Deferred@1718834912 after retry RPC after error@70569937)))
21:04:03.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.278 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@615970867(state=PENDING, result=null, callback=(continuation of Deferred@2048721135 after retry RPC after error@1836133792) -> (continuation of Deferred@1368645291 after retry RPC after error@1123533770) -> (continuation of Deferred@1350543619 after retry RPC after error@872907393) -> (continuation of Deferred@2075030572 after retry RPC after error@129696000) -> (continuation of Deferred@715233093 after retry RPC after error@574233370) -> (continuation of Deferred@708641893 after retry RPC after error@1976677155) -> (continuation of Deferred@1399593315 after retry RPC after error@561785567), errback=(continuation of Deferred@2048721135 after retry RPC after error@1836133792) -> (continuation of Deferred@1368645291 after retry RPC after error@1123533770) -> (continuation of Deferred@1350543619 after retry RPC after error@872907393) -> (continuation of Deferred@2075030572 after retry RPC after error@129696000) -> (continuation of Deferred@715233093 after retry RPC after error@574233370) -> (continuation of Deferred@708641893 after retry RPC after error@1976677155) -> (continuation of Deferred@1399593315 after retry RPC after error@561785567)))
21:04:03.278 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:03.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.340 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.340 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.557 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@539928361(state=PENDING, result=null, callback=(continuation of Deferred@2076350448 after retry RPC after error@196942285) -> (continuation of Deferred@732901451 after retry RPC after error@2129513976) -> (continuation of Deferred@301967554 after retry RPC after error@1928308801) -> (continuation of Deferred@546366271 after retry RPC after error@805464280) -> (continuation of Deferred@1549234082 after retry RPC after error@1408596425) -> (continuation of Deferred@1960571283 after retry RPC after error@1471943544) -> (continuation of Deferred@821120082 after retry RPC after error@1219641524), errback=(continuation of Deferred@2076350448 after retry RPC after error@196942285) -> (continuation of Deferred@732901451 after retry RPC after error@2129513976) -> (continuation of Deferred@301967554 after retry RPC after error@1928308801) -> (continuation of Deferred@546366271 after retry RPC after error@805464280) -> (continuation of Deferred@1549234082 after retry RPC after error@1408596425) -> (continuation of Deferred@1960571283 after retry RPC after error@1471943544) -> (continuation of Deferred@821120082 after retry RPC after error@1219641524)))
21:04:03.558 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:03.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.660 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.697 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.697 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.779 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@565853300(state=PENDING, result=null, callback=(continuation of Deferred@1423145417 after retry RPC after error@298378007) -> (continuation of Deferred@277400704 after retry RPC after error@1621166969) -> (continuation of Deferred@804465557 after retry RPC after error@2046141735) -> (continuation of Deferred@793746005 after retry RPC after error@1205712390) -> (continuation of Deferred@1908703979 after retry RPC after error@625790313) -> (continuation of Deferred@1979868839 after retry RPC after error@1548063719), errback=(continuation of Deferred@1423145417 after retry RPC after error@298378007) -> (continuation of Deferred@277400704 after retry RPC after error@1621166969) -> (continuation of Deferred@804465557 after retry RPC after error@2046141735) -> (continuation of Deferred@793746005 after retry RPC after error@1205712390) -> (continuation of Deferred@1908703979 after retry RPC after error@625790313) -> (continuation of Deferred@1979868839 after retry RPC after error@1548063719)))
21:04:03.780 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@565853300(state=PENDING, result=null, callback=(continuation of Deferred@1423145417 after retry RPC after error@298378007) -> (continuation of Deferred@277400704 after retry RPC after error@1621166969) -> (continuation of Deferred@804465557 after retry RPC after error@2046141735) -> (continuation of Deferred@793746005 after retry RPC after error@1205712390) -> (continuation of Deferred@1908703979 after retry RPC after error@625790313) -> (continuation of Deferred@1979868839 after retry RPC after error@1548063719), errback=(continuation of Deferred@1423145417 after retry RPC after error@298378007) -> (continuation of Deferred@277400704 after retry RPC after error@1621166969) -> (continuation of Deferred@804465557 after retry RPC after error@2046141735) -> (continuation of Deferred@793746005 after retry RPC after error@1205712390) -> (continuation of Deferred@1908703979 after retry RPC after error@625790313) -> (continuation of Deferred@1979868839 after retry RPC after error@1548063719)))
21:04:03.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:03.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:03.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:03.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:03.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:03.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.039 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.040 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.040 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1225815520(state=PENDING, result=null, callback=(continuation of Deferred@2095858492 after retry RPC after error@1263877070) -> (continuation of Deferred@2030968673 after retry RPC after error@863754965) -> (continuation of Deferred@756062757 after retry RPC after error@1341742850) -> (continuation of Deferred@1174779019 after retry RPC after error@734356460) -> (continuation of Deferred@44897539 after retry RPC after error@120146715) -> (continuation of Deferred@1826817689 after retry RPC after error@1815728874) -> (continuation of Deferred@114743139 after retry RPC after error@1273636145), errback=(continuation of Deferred@2095858492 after retry RPC after error@1263877070) -> (continuation of Deferred@2030968673 after retry RPC after error@863754965) -> (continuation of Deferred@756062757 after retry RPC after error@1341742850) -> (continuation of Deferred@1174779019 after retry RPC after error@734356460) -> (continuation of Deferred@44897539 after retry RPC after error@120146715) -> (continuation of Deferred@1826817689 after retry RPC after error@1815728874) -> (continuation of Deferred@114743139 after retry RPC after error@1273636145)))
21:04:04.041 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1225815520(state=PENDING, result=null, callback=(continuation of Deferred@2095858492 after retry RPC after error@1263877070) -> (continuation of Deferred@2030968673 after retry RPC after error@863754965) -> (continuation of Deferred@756062757 after retry RPC after error@1341742850) -> (continuation of Deferred@1174779019 after retry RPC after error@734356460) -> (continuation of Deferred@44897539 after retry RPC after error@120146715) -> (continuation of Deferred@1826817689 after retry RPC after error@1815728874) -> (continuation of Deferred@114743139 after retry RPC after error@1273636145), errback=(continuation of Deferred@2095858492 after retry RPC after error@1263877070) -> (continuation of Deferred@2030968673 after retry RPC after error@863754965) -> (continuation of Deferred@756062757 after retry RPC after error@1341742850) -> (continuation of Deferred@1174779019 after retry RPC after error@734356460) -> (continuation of Deferred@44897539 after retry RPC after error@120146715) -> (continuation of Deferred@1826817689 after retry RPC after error@1815728874) -> (continuation of Deferred@114743139 after retry RPC after error@1273636145)))
21:04:04.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:04.113911 4289 raft_consensus.cc:491] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 368a4202fe704bd8addedbf58359d23f)
21:04:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:04.114130 4289 raft_consensus.cc:513] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "368a4202fe704bd8addedbf58359d23f" member_type: VOTER last_known_addr { host: "127.3.35.126" port: 35753 } } peers { permanent_uuid: "fb03beba58e94739ba9391a7dac8aac8" member_type: VOTER last_known_addr { host: "127.3.35.125" port: 42683 } } peers { permanent_uuid: "c8e1b47b9a32447d8758d067e7a5dd60" member_type: VOTER last_known_addr { host: "127.3.35.124" port: 46649 } }
21:04:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:04.114831 4289 leader_election.cc:290] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753), c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649)
21:04:04.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:04.116009 3322 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111) [suppressed 1 similar messages]
21:04:04.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:04.117261 3320 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 368a4202fe704bd8addedbf58359d23f (127.3.35.126:35753): Network error: Client connection negotiation failed: client connection to 127.3.35.126:35753: connect: Connection refused (error 111)
21:04:04.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:04.117452 3322 leader_election.cc:336] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c8e1b47b9a32447d8758d067e7a5dd60 (127.3.35.124:46649): Network error: Client connection negotiation failed: client connection to 127.3.35.124:46649: connect: Connection refused (error 111)
21:04:04.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:04.117568 3322 leader_election.cc:304] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fb03beba58e94739ba9391a7dac8aac8; no voters: 368a4202fe704bd8addedbf58359d23f, c8e1b47b9a32447d8758d067e7a5dd60
21:04:04.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:04.117822 4289 raft_consensus.cc:2743] T 00000000000000000000000000000000 P fb03beba58e94739ba9391a7dac8aac8 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:04:04.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.220 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1272034566(state=PENDING, result=null, callback=(continuation of Deferred@636899063 after retry RPC after error@67483115) -> (continuation of Deferred@507759460 after retry RPC after error@1484534831) -> (continuation of Deferred@1332365345 after retry RPC after error@1726551754) -> (continuation of Deferred@1727134104 after retry RPC after error@1603630117) -> (continuation of Deferred@1109595318 after retry RPC after error@1825707390) -> (continuation of Deferred@92675141 after retry RPC after error@1900442764), errback=(continuation of Deferred@636899063 after retry RPC after error@67483115) -> (continuation of Deferred@507759460 after retry RPC after error@1484534831) -> (continuation of Deferred@1332365345 after retry RPC after error@1726551754) -> (continuation of Deferred@1727134104 after retry RPC after error@1603630117) -> (continuation of Deferred@1109595318 after retry RPC after error@1825707390) -> (continuation of Deferred@92675141 after retry RPC after error@1900442764)))
21:04:04.220 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1272034566(state=PENDING, result=null, callback=(continuation of Deferred@636899063 after retry RPC after error@67483115) -> (continuation of Deferred@507759460 after retry RPC after error@1484534831) -> (continuation of Deferred@1332365345 after retry RPC after error@1726551754) -> (continuation of Deferred@1727134104 after retry RPC after error@1603630117) -> (continuation of Deferred@1109595318 after retry RPC after error@1825707390) -> (continuation of Deferred@92675141 after retry RPC after error@1900442764), errback=(continuation of Deferred@636899063 after retry RPC after error@67483115) -> (continuation of Deferred@507759460 after retry RPC after error@1484534831) -> (continuation of Deferred@1332365345 after retry RPC after error@1726551754) -> (continuation of Deferred@1727134104 after retry RPC after error@1603630117) -> (continuation of Deferred@1109595318 after retry RPC after error@1825707390) -> (continuation of Deferred@92675141 after retry RPC after error@1900442764)))
21:04:04.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.498 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@77691582(state=PENDING, result=null, callback=(continuation of Deferred@1538652563 after retry RPC after error@1221077804) -> (continuation of Deferred@635746358 after retry RPC after error@715084478) -> (continuation of Deferred@1586650911 after retry RPC after error@1460313770) -> (continuation of Deferred@1471129973 after retry RPC after error@133499804) -> (continuation of Deferred@1272170202 after retry RPC after error@1219493572) -> (continuation of Deferred@1650524837 after retry RPC after error@1298619550) -> (continuation of Deferred@408397566 after retry RPC after error@800666350), errback=(continuation of Deferred@1538652563 after retry RPC after error@1221077804) -> (continuation of Deferred@635746358 after retry RPC after error@715084478) -> (continuation of Deferred@1586650911 after retry RPC after error@1460313770) -> (continuation of Deferred@1471129973 after retry RPC after error@133499804) -> (continuation of Deferred@1272170202 after retry RPC after error@1219493572) -> (continuation of Deferred@1650524837 after retry RPC after error@1298619550) -> (continuation of Deferred@408397566 after retry RPC after error@800666350)))
21:04:04.498 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:04.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.680 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.720 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1985820546(state=PENDING, result=null, callback=(continuation of Deferred@1944591248 after retry RPC after error@1166603956) -> (continuation of Deferred@1594986761 after retry RPC after error@1089424462) -> (continuation of Deferred@1936259928 after retry RPC after error@1909956311) -> (continuation of Deferred@572375101 after retry RPC after error@405431564) -> (continuation of Deferred@749530986 after retry RPC after error@1640033550) -> (continuation of Deferred@764466467 after retry RPC after error@1725552536) -> (continuation of Deferred@759542678 after retry RPC after error@1267434413), errback=(continuation of Deferred@1944591248 after retry RPC after error@1166603956) -> (continuation of Deferred@1594986761 after retry RPC after error@1089424462) -> (continuation of Deferred@1936259928 after retry RPC after error@1909956311) -> (continuation of Deferred@572375101 after retry RPC after error@405431564) -> (continuation of Deferred@749530986 after retry RPC after error@1640033550) -> (continuation of Deferred@764466467 after retry RPC after error@1725552536) -> (continuation of Deferred@759542678 after retry RPC after error@1267434413)))
21:04:04.720 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1985820546(state=PENDING, result=null, callback=(continuation of Deferred@1944591248 after retry RPC after error@1166603956) -> (continuation of Deferred@1594986761 after retry RPC after error@1089424462) -> (continuation of Deferred@1936259928 after retry RPC after error@1909956311) -> (continuation of Deferred@572375101 after retry RPC after error@405431564) -> (continuation of Deferred@749530986 after retry RPC after error@1640033550) -> (continuation of Deferred@764466467 after retry RPC after error@1725552536) -> (continuation of Deferred@759542678 after retry RPC after error@1267434413), errback=(continuation of Deferred@1944591248 after retry RPC after error@1166603956) -> (continuation of Deferred@1594986761 after retry RPC after error@1089424462) -> (continuation of Deferred@1936259928 after retry RPC after error@1909956311) -> (continuation of Deferred@572375101 after retry RPC after error@405431564) -> (continuation of Deferred@749530986 after retry RPC after error@1640033550) -> (continuation of Deferred@764466467 after retry RPC after error@1725552536) -> (continuation of Deferred@759542678 after retry RPC after error@1267434413)))
21:04:04.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.737 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@574402790(state=PENDING, result=null, callback=(continuation of Deferred@1285253985 after retry RPC after error@153265010) -> (continuation of Deferred@1850351425 after retry RPC after error@691419630) -> (continuation of Deferred@1046585290 after retry RPC after error@583303751) -> (continuation of Deferred@483008841 after retry RPC after error@78348919) -> (continuation of Deferred@914805479 after retry RPC after error@810985674) -> (continuation of Deferred@2103121888 after retry RPC after error@633051654), errback=(continuation of Deferred@1285253985 after retry RPC after error@153265010) -> (continuation of Deferred@1850351425 after retry RPC after error@691419630) -> (continuation of Deferred@1046585290 after retry RPC after error@583303751) -> (continuation of Deferred@483008841 after retry RPC after error@78348919) -> (continuation of Deferred@914805479 after retry RPC after error@810985674) -> (continuation of Deferred@2103121888 after retry RPC after error@633051654)))
21:04:04.880 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@574402790(state=PENDING, result=null, callback=(continuation of Deferred@1285253985 after retry RPC after error@153265010) -> (continuation of Deferred@1850351425 after retry RPC after error@691419630) -> (continuation of Deferred@1046585290 after retry RPC after error@583303751) -> (continuation of Deferred@483008841 after retry RPC after error@78348919) -> (continuation of Deferred@914805479 after retry RPC after error@810985674) -> (continuation of Deferred@2103121888 after retry RPC after error@633051654), errback=(continuation of Deferred@1285253985 after retry RPC after error@153265010) -> (continuation of Deferred@1850351425 after retry RPC after error@691419630) -> (continuation of Deferred@1046585290 after retry RPC after error@583303751) -> (continuation of Deferred@483008841 after retry RPC after error@78348919) -> (continuation of Deferred@914805479 after retry RPC after error@810985674) -> (continuation of Deferred@2103121888 after retry RPC after error@633051654)))
21:04:04.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.899 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:04.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:04.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:04.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:04.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:04.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.060 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@637046631(state=PENDING, result=null, callback=(continuation of Deferred@1951322655 after retry RPC after error@1595175566) -> (continuation of Deferred@1969314886 after retry RPC after error@344155706) -> (continuation of Deferred@383750303 after retry RPC after error@1780678856) -> (continuation of Deferred@366998182 after retry RPC after error@442688657) -> (continuation of Deferred@1884537898 after retry RPC after error@42254248) -> (continuation of Deferred@1755592604 after retry RPC after error@1701777525), errback=(continuation of Deferred@1951322655 after retry RPC after error@1595175566) -> (continuation of Deferred@1969314886 after retry RPC after error@344155706) -> (continuation of Deferred@383750303 after retry RPC after error@1780678856) -> (continuation of Deferred@366998182 after retry RPC after error@442688657) -> (continuation of Deferred@1884537898 after retry RPC after error@42254248) -> (continuation of Deferred@1755592604 after retry RPC after error@1701777525)))
21:04:05.060 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@637046631(state=PENDING, result=null, callback=(continuation of Deferred@1951322655 after retry RPC after error@1595175566) -> (continuation of Deferred@1969314886 after retry RPC after error@344155706) -> (continuation of Deferred@383750303 after retry RPC after error@1780678856) -> (continuation of Deferred@366998182 after retry RPC after error@442688657) -> (continuation of Deferred@1884537898 after retry RPC after error@42254248) -> (continuation of Deferred@1755592604 after retry RPC after error@1701777525), errback=(continuation of Deferred@1951322655 after retry RPC after error@1595175566) -> (continuation of Deferred@1969314886 after retry RPC after error@344155706) -> (continuation of Deferred@383750303 after retry RPC after error@1780678856) -> (continuation of Deferred@366998182 after retry RPC after error@442688657) -> (continuation of Deferred@1884537898 after retry RPC after error@42254248) -> (continuation of Deferred@1755592604 after retry RPC after error@1701777525)))
21:04:05.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.098 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.239 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@175404125(state=PENDING, result=null, callback=(continuation of Deferred@1044063766 after retry RPC after error@1539932170) -> (continuation of Deferred@1676351695 after retry RPC after error@1242844787) -> (continuation of Deferred@771279537 after retry RPC after error@984012500) -> (continuation of Deferred@19958831 after retry RPC after error@1765752236) -> (continuation of Deferred@41541496 after retry RPC after error@249932181) -> (continuation of Deferred@187764114 after retry RPC after error@460559109), errback=(continuation of Deferred@1044063766 after retry RPC after error@1539932170) -> (continuation of Deferred@1676351695 after retry RPC after error@1242844787) -> (continuation of Deferred@771279537 after retry RPC after error@984012500) -> (continuation of Deferred@19958831 after retry RPC after error@1765752236) -> (continuation of Deferred@41541496 after retry RPC after error@249932181) -> (continuation of Deferred@187764114 after retry RPC after error@460559109)))
21:04:05.240 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@175404125(state=PENDING, result=null, callback=(continuation of Deferred@1044063766 after retry RPC after error@1539932170) -> (continuation of Deferred@1676351695 after retry RPC after error@1242844787) -> (continuation of Deferred@771279537 after retry RPC after error@984012500) -> (continuation of Deferred@19958831 after retry RPC after error@1765752236) -> (continuation of Deferred@41541496 after retry RPC after error@249932181) -> (continuation of Deferred@187764114 after retry RPC after error@460559109), errback=(continuation of Deferred@1044063766 after retry RPC after error@1539932170) -> (continuation of Deferred@1676351695 after retry RPC after error@1242844787) -> (continuation of Deferred@771279537 after retry RPC after error@984012500) -> (continuation of Deferred@19958831 after retry RPC after error@1765752236) -> (continuation of Deferred@41541496 after retry RPC after error@249932181) -> (continuation of Deferred@187764114 after retry RPC after error@460559109)))
21:04:05.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.440 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1678852803(state=PENDING, result=null, callback=(continuation of Deferred@11294518 after retry RPC after error@1917223471) -> (continuation of Deferred@839497103 after retry RPC after error@1558249047) -> (continuation of Deferred@1050085146 after retry RPC after error@2013703453) -> (continuation of Deferred@123525727 after retry RPC after error@1544916645) -> (continuation of Deferred@1513946602 after retry RPC after error@396348917) -> (continuation of Deferred@1995465595 after retry RPC after error@107002790) -> (continuation of Deferred@999212333 after retry RPC after error@1385650124), errback=(continuation of Deferred@11294518 after retry RPC after error@1917223471) -> (continuation of Deferred@839497103 after retry RPC after error@1558249047) -> (continuation of Deferred@1050085146 after retry RPC after error@2013703453) -> (continuation of Deferred@123525727 after retry RPC after error@1544916645) -> (continuation of Deferred@1513946602 after retry RPC after error@396348917) -> (continuation of Deferred@1995465595 after retry RPC after error@107002790) -> (continuation of Deferred@999212333 after retry RPC after error@1385650124)))
21:04:05.440 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1678852803(state=PENDING, result=null, callback=(continuation of Deferred@11294518 after retry RPC after error@1917223471) -> (continuation of Deferred@839497103 after retry RPC after error@1558249047) -> (continuation of Deferred@1050085146 after retry RPC after error@2013703453) -> (continuation of Deferred@123525727 after retry RPC after error@1544916645) -> (continuation of Deferred@1513946602 after retry RPC after error@396348917) -> (continuation of Deferred@1995465595 after retry RPC after error@107002790) -> (continuation of Deferred@999212333 after retry RPC after error@1385650124), errback=(continuation of Deferred@11294518 after retry RPC after error@1917223471) -> (continuation of Deferred@839497103 after retry RPC after error@1558249047) -> (continuation of Deferred@1050085146 after retry RPC after error@2013703453) -> (continuation of Deferred@123525727 after retry RPC after error@1544916645) -> (continuation of Deferred@1513946602 after retry RPC after error@396348917) -> (continuation of Deferred@1995465595 after retry RPC after error@107002790) -> (continuation of Deferred@999212333 after retry RPC after error@1385650124)))
21:04:05.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.640 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.640 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2016573610(state=PENDING, result=null, callback=(continuation of Deferred@1186173193 after retry RPC after error@1286423433) -> (continuation of Deferred@1405529303 after retry RPC after error@945453357) -> (continuation of Deferred@1648073974 after retry RPC after error@1152133683) -> (continuation of Deferred@577410661 after retry RPC after error@1225920532) -> (continuation of Deferred@2003892716 after retry RPC after error@1386788995) -> (continuation of Deferred@604493069 after retry RPC after error@2110583593) -> (continuation of Deferred@226745061 after retry RPC after error@452462454), errback=(continuation of Deferred@1186173193 after retry RPC after error@1286423433) -> (continuation of Deferred@1405529303 after retry RPC after error@945453357) -> (continuation of Deferred@1648073974 after retry RPC after error@1152133683) -> (continuation of Deferred@577410661 after retry RPC after error@1225920532) -> (continuation of Deferred@2003892716 after retry RPC after error@1386788995) -> (continuation of Deferred@604493069 after retry RPC after error@2110583593) -> (continuation of Deferred@226745061 after retry RPC after error@452462454)))
21:04:05.641 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2016573610(state=PENDING, result=null, callback=(continuation of Deferred@1186173193 after retry RPC after error@1286423433) -> (continuation of Deferred@1405529303 after retry RPC after error@945453357) -> (continuation of Deferred@1648073974 after retry RPC after error@1152133683) -> (continuation of Deferred@577410661 after retry RPC after error@1225920532) -> (continuation of Deferred@2003892716 after retry RPC after error@1386788995) -> (continuation of Deferred@604493069 after retry RPC after error@2110583593) -> (continuation of Deferred@226745061 after retry RPC after error@452462454), errback=(continuation of Deferred@1186173193 after retry RPC after error@1286423433) -> (continuation of Deferred@1405529303 after retry RPC after error@945453357) -> (continuation of Deferred@1648073974 after retry RPC after error@1152133683) -> (continuation of Deferred@577410661 after retry RPC after error@1225920532) -> (continuation of Deferred@2003892716 after retry RPC after error@1386788995) -> (continuation of Deferred@604493069 after retry RPC after error@2110583593) -> (continuation of Deferred@226745061 after retry RPC after error@452462454)))
21:04:05.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.678 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.717 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.717 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.819 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.917 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2011422615(state=PENDING, result=null, callback=(continuation of Deferred@1156246863 after retry RPC after error@1714020200) -> (continuation of Deferred@113259547 after retry RPC after error@1874222131) -> (continuation of Deferred@2091209759 after retry RPC after error@1423906428) -> (continuation of Deferred@1396211728 after retry RPC after error@238164352) -> (continuation of Deferred@1482797557 after retry RPC after error@1948328051) -> (continuation of Deferred@1670256359 after retry RPC after error@2112810255) -> (continuation of Deferred@676664670 after retry RPC after error@1426754207), errback=(continuation of Deferred@1156246863 after retry RPC after error@1714020200) -> (continuation of Deferred@113259547 after retry RPC after error@1874222131) -> (continuation of Deferred@2091209759 after retry RPC after error@1423906428) -> (continuation of Deferred@1396211728 after retry RPC after error@238164352) -> (continuation of Deferred@1482797557 after retry RPC after error@1948328051) -> (continuation of Deferred@1670256359 after retry RPC after error@2112810255) -> (continuation of Deferred@676664670 after retry RPC after error@1426754207)))
21:04:05.918 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:05.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.939 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.940 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:05.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:05.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:05.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:05.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:05.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.058 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.140 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@734893577(state=PENDING, result=null, callback=(continuation of Deferred@63565014 after retry RPC after error@543695779) -> (continuation of Deferred@205800191 after retry RPC after error@1274998944) -> (continuation of Deferred@119343701 after retry RPC after error@1711883310) -> (continuation of Deferred@1223991722 after retry RPC after error@1949512676) -> (continuation of Deferred@462672021 after retry RPC after error@932700157) -> (continuation of Deferred@1510195540 after retry RPC after error@927098339), errback=(continuation of Deferred@63565014 after retry RPC after error@543695779) -> (continuation of Deferred@205800191 after retry RPC after error@1274998944) -> (continuation of Deferred@119343701 after retry RPC after error@1711883310) -> (continuation of Deferred@1223991722 after retry RPC after error@1949512676) -> (continuation of Deferred@462672021 after retry RPC after error@932700157) -> (continuation of Deferred@1510195540 after retry RPC after error@927098339)))
21:04:06.140 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@734893577(state=PENDING, result=null, callback=(continuation of Deferred@63565014 after retry RPC after error@543695779) -> (continuation of Deferred@205800191 after retry RPC after error@1274998944) -> (continuation of Deferred@119343701 after retry RPC after error@1711883310) -> (continuation of Deferred@1223991722 after retry RPC after error@1949512676) -> (continuation of Deferred@462672021 after retry RPC after error@932700157) -> (continuation of Deferred@1510195540 after retry RPC after error@927098339), errback=(continuation of Deferred@63565014 after retry RPC after error@543695779) -> (continuation of Deferred@205800191 after retry RPC after error@1274998944) -> (continuation of Deferred@119343701 after retry RPC after error@1711883310) -> (continuation of Deferred@1223991722 after retry RPC after error@1949512676) -> (continuation of Deferred@462672021 after retry RPC after error@932700157) -> (continuation of Deferred@1510195540 after retry RPC after error@927098339)))
21:04:06.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.220 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.220 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.239 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.278 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.339 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@636402865(state=PENDING, result=null, callback=(continuation of Deferred@2064382740 after retry RPC after error@355894567) -> (continuation of Deferred@265711567 after retry RPC after error@1594708400) -> (continuation of Deferred@2071642654 after retry RPC after error@1514434539) -> (continuation of Deferred@1716575099 after retry RPC after error@1399166672) -> (continuation of Deferred@2124596368 after retry RPC after error@132175768) -> (continuation of Deferred@1728794396 after retry RPC after error@413636155) -> (continuation of Deferred@1892239355 after retry RPC after error@1897136749), errback=(continuation of Deferred@2064382740 after retry RPC after error@355894567) -> (continuation of Deferred@265711567 after retry RPC after error@1594708400) -> (continuation of Deferred@2071642654 after retry RPC after error@1514434539) -> (continuation of Deferred@1716575099 after retry RPC after error@1399166672) -> (continuation of Deferred@2124596368 after retry RPC after error@132175768) -> (continuation of Deferred@1728794396 after retry RPC after error@413636155) -> (continuation of Deferred@1892239355 after retry RPC after error@1897136749)))
21:04:06.340 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@636402865(state=PENDING, result=null, callback=(continuation of Deferred@2064382740 after retry RPC after error@355894567) -> (continuation of Deferred@265711567 after retry RPC after error@1594708400) -> (continuation of Deferred@2071642654 after retry RPC after error@1514434539) -> (continuation of Deferred@1716575099 after retry RPC after error@1399166672) -> (continuation of Deferred@2124596368 after retry RPC after error@132175768) -> (continuation of Deferred@1728794396 after retry RPC after error@413636155) -> (continuation of Deferred@1892239355 after retry RPC after error@1897136749), errback=(continuation of Deferred@2064382740 after retry RPC after error@355894567) -> (continuation of Deferred@265711567 after retry RPC after error@1594708400) -> (continuation of Deferred@2071642654 after retry RPC after error@1514434539) -> (continuation of Deferred@1716575099 after retry RPC after error@1399166672) -> (continuation of Deferred@2124596368 after retry RPC after error@132175768) -> (continuation of Deferred@1728794396 after retry RPC after error@413636155) -> (continuation of Deferred@1892239355 after retry RPC after error@1897136749)))
21:04:06.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.579 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@838047479(state=PENDING, result=null, callback=(continuation of Deferred@591503044 after retry RPC after error@900822108) -> (continuation of Deferred@754780682 after retry RPC after error@26502476) -> (continuation of Deferred@772566020 after retry RPC after error@270659085) -> (continuation of Deferred@1414011737 after retry RPC after error@1985264281) -> (continuation of Deferred@997561371 after retry RPC after error@607864062) -> (continuation of Deferred@809304563 after retry RPC after error@1668211999) -> (continuation of Deferred@1949069349 after retry RPC after error@181323498), errback=(continuation of Deferred@591503044 after retry RPC after error@900822108) -> (continuation of Deferred@754780682 after retry RPC after error@26502476) -> (continuation of Deferred@772566020 after retry RPC after error@270659085) -> (continuation of Deferred@1414011737 after retry RPC after error@1985264281) -> (continuation of Deferred@997561371 after retry RPC after error@607864062) -> (continuation of Deferred@809304563 after retry RPC after error@1668211999) -> (continuation of Deferred@1949069349 after retry RPC after error@181323498)))
21:04:06.580 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@838047479(state=PENDING, result=null, callback=(continuation of Deferred@591503044 after retry RPC after error@900822108) -> (continuation of Deferred@754780682 after retry RPC after error@26502476) -> (continuation of Deferred@772566020 after retry RPC after error@270659085) -> (continuation of Deferred@1414011737 after retry RPC after error@1985264281) -> (continuation of Deferred@997561371 after retry RPC after error@607864062) -> (continuation of Deferred@809304563 after retry RPC after error@1668211999) -> (continuation of Deferred@1949069349 after retry RPC after error@181323498), errback=(continuation of Deferred@591503044 after retry RPC after error@900822108) -> (continuation of Deferred@754780682 after retry RPC after error@26502476) -> (continuation of Deferred@772566020 after retry RPC after error@270659085) -> (continuation of Deferred@1414011737 after retry RPC after error@1985264281) -> (continuation of Deferred@997561371 after retry RPC after error@607864062) -> (continuation of Deferred@809304563 after retry RPC after error@1668211999) -> (continuation of Deferred@1949069349 after retry RPC after error@181323498)))
21:04:06.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.620 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.640 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.760 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2133006751(state=PENDING, result=null, callback=(continuation of Deferred@2045369640 after retry RPC after error@720188793) -> (continuation of Deferred@193822279 after retry RPC after error@1812296008) -> (continuation of Deferred@1530061181 after retry RPC after error@406278559) -> (continuation of Deferred@2015914904 after retry RPC after error@1071700708) -> (continuation of Deferred@1930816876 after retry RPC after error@1359798584) -> (continuation of Deferred@1782789162 after retry RPC after error@1397617788) -> (continuation of Deferred@258936308 after retry RPC after error@962340864), errback=(continuation of Deferred@2045369640 after retry RPC after error@720188793) -> (continuation of Deferred@193822279 after retry RPC after error@1812296008) -> (continuation of Deferred@1530061181 after retry RPC after error@406278559) -> (continuation of Deferred@2015914904 after retry RPC after error@1071700708) -> (continuation of Deferred@1930816876 after retry RPC after error@1359798584) -> (continuation of Deferred@1782789162 after retry RPC after error@1397617788) -> (continuation of Deferred@258936308 after retry RPC after error@962340864)))
21:04:06.760 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2133006751(state=PENDING, result=null, callback=(continuation of Deferred@2045369640 after retry RPC after error@720188793) -> (continuation of Deferred@193822279 after retry RPC after error@1812296008) -> (continuation of Deferred@1530061181 after retry RPC after error@406278559) -> (continuation of Deferred@2015914904 after retry RPC after error@1071700708) -> (continuation of Deferred@1930816876 after retry RPC after error@1359798584) -> (continuation of Deferred@1782789162 after retry RPC after error@1397617788) -> (continuation of Deferred@258936308 after retry RPC after error@962340864), errback=(continuation of Deferred@2045369640 after retry RPC after error@720188793) -> (continuation of Deferred@193822279 after retry RPC after error@1812296008) -> (continuation of Deferred@1530061181 after retry RPC after error@406278559) -> (continuation of Deferred@2015914904 after retry RPC after error@1071700708) -> (continuation of Deferred@1930816876 after retry RPC after error@1359798584) -> (continuation of Deferred@1782789162 after retry RPC after error@1397617788) -> (continuation of Deferred@258936308 after retry RPC after error@962340864)))
21:04:06.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.917 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.917 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.918 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:06.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:06.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:06.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:06.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:06.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.038 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@562275719(state=PENDING, result=null, callback=(continuation of Deferred@1811654074 after retry RPC after error@1807981156) -> (continuation of Deferred@437700232 after retry RPC after error@637921232) -> (continuation of Deferred@283459807 after retry RPC after error@1736632349) -> (continuation of Deferred@1267482851 after retry RPC after error@1119753509) -> (continuation of Deferred@2020034069 after retry RPC after error@148212484) -> (continuation of Deferred@561469861 after retry RPC after error@332560275) -> (continuation of Deferred@1372052498 after retry RPC after error@1978589317) -> (continuation of Deferred@1859069420 after retry RPC after error@1856092489), errback=(continuation of Deferred@1811654074 after retry RPC after error@1807981156) -> (continuation of Deferred@437700232 after retry RPC after error@637921232) -> (continuation of Deferred@283459807 after retry RPC after error@1736632349) -> (continuation of Deferred@1267482851 after retry RPC after error@1119753509) -> (continuation of Deferred@2020034069 after retry RPC after error@148212484) -> (continuation of Deferred@561469861 after retry RPC after error@332560275) -> (continuation of Deferred@1372052498 after retry RPC after error@1978589317) -> (continuation of Deferred@1859069420 after retry RPC after error@1856092489)))
21:04:07.038 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:07.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.077 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.098 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.299 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1863835224(state=PENDING, result=null, callback=(continuation of Deferred@1171937470 after retry RPC after error@301918304) -> (continuation of Deferred@1757766698 after retry RPC after error@963740276) -> (continuation of Deferred@779690572 after retry RPC after error@1210212875) -> (continuation of Deferred@184924945 after retry RPC after error@1045621287) -> (continuation of Deferred@1499387107 after retry RPC after error@37210105) -> (continuation of Deferred@608975261 after retry RPC after error@224849121) -> (continuation of Deferred@582015074 after retry RPC after error@512023490), errback=(continuation of Deferred@1171937470 after retry RPC after error@301918304) -> (continuation of Deferred@1757766698 after retry RPC after error@963740276) -> (continuation of Deferred@779690572 after retry RPC after error@1210212875) -> (continuation of Deferred@184924945 after retry RPC after error@1045621287) -> (continuation of Deferred@1499387107 after retry RPC after error@37210105) -> (continuation of Deferred@608975261 after retry RPC after error@224849121) -> (continuation of Deferred@582015074 after retry RPC after error@512023490)))
21:04:07.300 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1863835224(state=PENDING, result=null, callback=(continuation of Deferred@1171937470 after retry RPC after error@301918304) -> (continuation of Deferred@1757766698 after retry RPC after error@963740276) -> (continuation of Deferred@779690572 after retry RPC after error@1210212875) -> (continuation of Deferred@184924945 after retry RPC after error@1045621287) -> (continuation of Deferred@1499387107 after retry RPC after error@37210105) -> (continuation of Deferred@608975261 after retry RPC after error@224849121) -> (continuation of Deferred@582015074 after retry RPC after error@512023490), errback=(continuation of Deferred@1171937470 after retry RPC after error@301918304) -> (continuation of Deferred@1757766698 after retry RPC after error@963740276) -> (continuation of Deferred@779690572 after retry RPC after error@1210212875) -> (continuation of Deferred@184924945 after retry RPC after error@1045621287) -> (continuation of Deferred@1499387107 after retry RPC after error@37210105) -> (continuation of Deferred@608975261 after retry RPC after error@224849121) -> (continuation of Deferred@582015074 after retry RPC after error@512023490)))
21:04:07.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.340 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.418 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.498 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.560 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@314524369(state=PENDING, result=null, callback=(continuation of Deferred@1641744778 after retry RPC after error@2097479888) -> (continuation of Deferred@780427716 after retry RPC after error@1178933007) -> (continuation of Deferred@2137768212 after retry RPC after error@708660649) -> (continuation of Deferred@658146155 after retry RPC after error@1227425260) -> (continuation of Deferred@1751846588 after retry RPC after error@687560221) -> (continuation of Deferred@1537849778 after retry RPC after error@1521402190) -> (continuation of Deferred@1786784329 after retry RPC after error@414827194) -> (continuation of Deferred@239836214 after retry RPC after error@574851072), errback=(continuation of Deferred@1641744778 after retry RPC after error@2097479888) -> (continuation of Deferred@780427716 after retry RPC after error@1178933007) -> (continuation of Deferred@2137768212 after retry RPC after error@708660649) -> (continuation of Deferred@658146155 after retry RPC after error@1227425260) -> (continuation of Deferred@1751846588 after retry RPC after error@687560221) -> (continuation of Deferred@1537849778 after retry RPC after error@1521402190) -> (continuation of Deferred@1786784329 after retry RPC after error@414827194) -> (continuation of Deferred@239836214 after retry RPC after error@574851072)))
21:04:07.560 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@314524369(state=PENDING, result=null, callback=(continuation of Deferred@1641744778 after retry RPC after error@2097479888) -> (continuation of Deferred@780427716 after retry RPC after error@1178933007) -> (continuation of Deferred@2137768212 after retry RPC after error@708660649) -> (continuation of Deferred@658146155 after retry RPC after error@1227425260) -> (continuation of Deferred@1751846588 after retry RPC after error@687560221) -> (continuation of Deferred@1537849778 after retry RPC after error@1521402190) -> (continuation of Deferred@1786784329 after retry RPC after error@414827194) -> (continuation of Deferred@239836214 after retry RPC after error@574851072), errback=(continuation of Deferred@1641744778 after retry RPC after error@2097479888) -> (continuation of Deferred@780427716 after retry RPC after error@1178933007) -> (continuation of Deferred@2137768212 after retry RPC after error@708660649) -> (continuation of Deferred@658146155 after retry RPC after error@1227425260) -> (continuation of Deferred@1751846588 after retry RPC after error@687560221) -> (continuation of Deferred@1537849778 after retry RPC after error@1521402190) -> (continuation of Deferred@1786784329 after retry RPC after error@414827194) -> (continuation of Deferred@239836214 after retry RPC after error@574851072)))
21:04:07.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.600 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.660 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.738 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.740 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2133302034(state=PENDING, result=null, callback=(continuation of Deferred@356799940 after retry RPC after error@2087255372) -> (continuation of Deferred@296583658 after retry RPC after error@1385260998) -> (continuation of Deferred@1707930149 after retry RPC after error@1584787789) -> (continuation of Deferred@589994424 after retry RPC after error@2052875596) -> (continuation of Deferred@2021916152 after retry RPC after error@151884860) -> (continuation of Deferred@2124229628 after retry RPC after error@1743935334), errback=(continuation of Deferred@356799940 after retry RPC after error@2087255372) -> (continuation of Deferred@296583658 after retry RPC after error@1385260998) -> (continuation of Deferred@1707930149 after retry RPC after error@1584787789) -> (continuation of Deferred@589994424 after retry RPC after error@2052875596) -> (continuation of Deferred@2021916152 after retry RPC after error@151884860) -> (continuation of Deferred@2124229628 after retry RPC after error@1743935334)))
21:04:07.740 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2133302034(state=PENDING, result=null, callback=(continuation of Deferred@356799940 after retry RPC after error@2087255372) -> (continuation of Deferred@296583658 after retry RPC after error@1385260998) -> (continuation of Deferred@1707930149 after retry RPC after error@1584787789) -> (continuation of Deferred@589994424 after retry RPC after error@2052875596) -> (continuation of Deferred@2021916152 after retry RPC after error@151884860) -> (continuation of Deferred@2124229628 after retry RPC after error@1743935334), errback=(continuation of Deferred@356799940 after retry RPC after error@2087255372) -> (continuation of Deferred@296583658 after retry RPC after error@1385260998) -> (continuation of Deferred@1707930149 after retry RPC after error@1584787789) -> (continuation of Deferred@589994424 after retry RPC after error@2052875596) -> (continuation of Deferred@2021916152 after retry RPC after error@151884860) -> (continuation of Deferred@2124229628 after retry RPC after error@1743935334)))
21:04:07.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:07.867779 3591 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:04:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:07.872300 3591 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '56f4ea9ce0054d5898b79cdba5d3c814', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.35.126:35753,127.3.35.125:42683,127.3.35.124:46649, num_attempts: 1)
21:04:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:07.873829 3591 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.35.124:46649)
21:04:07.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.940 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@884194364(state=PENDING, result=null, callback=(continuation of Deferred@618443499 after retry RPC after error@880789855) -> (continuation of Deferred@1160152345 after retry RPC after error@1818067933) -> (continuation of Deferred@760688471 after retry RPC after error@1730686611) -> (continuation of Deferred@374825250 after retry RPC after error@378817374) -> (continuation of Deferred@36561740 after retry RPC after error@609683756) -> (continuation of Deferred@161751797 after retry RPC after error@1029588516) -> (continuation of Deferred@1342498112 after retry RPC after error@1283885061), errback=(continuation of Deferred@618443499 after retry RPC after error@880789855) -> (continuation of Deferred@1160152345 after retry RPC after error@1818067933) -> (continuation of Deferred@760688471 after retry RPC after error@1730686611) -> (continuation of Deferred@374825250 after retry RPC after error@378817374) -> (continuation of Deferred@36561740 after retry RPC after error@609683756) -> (continuation of Deferred@161751797 after retry RPC after error@1029588516) -> (continuation of Deferred@1342498112 after retry RPC after error@1283885061)))
21:04:07.940 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@884194364(state=PENDING, result=null, callback=(continuation of Deferred@618443499 after retry RPC after error@880789855) -> (continuation of Deferred@1160152345 after retry RPC after error@1818067933) -> (continuation of Deferred@760688471 after retry RPC after error@1730686611) -> (continuation of Deferred@374825250 after retry RPC after error@378817374) -> (continuation of Deferred@36561740 after retry RPC after error@609683756) -> (continuation of Deferred@161751797 after retry RPC after error@1029588516) -> (continuation of Deferred@1342498112 after retry RPC after error@1283885061), errback=(continuation of Deferred@618443499 after retry RPC after error@880789855) -> (continuation of Deferred@1160152345 after retry RPC after error@1818067933) -> (continuation of Deferred@760688471 after retry RPC after error@1730686611) -> (continuation of Deferred@374825250 after retry RPC after error@378817374) -> (continuation of Deferred@36561740 after retry RPC after error@609683756) -> (continuation of Deferred@161751797 after retry RPC after error@1029588516) -> (continuation of Deferred@1342498112 after retry RPC after error@1283885061)))
21:04:07.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.958 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:07.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:07.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:07.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:07.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:07.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.018 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.157 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.160 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2021741937(state=PENDING, result=null, callback=(continuation of Deferred@413673262 after retry RPC after error@1328826370) -> (continuation of Deferred@939282732 after retry RPC after error@1160032383) -> (continuation of Deferred@485260962 after retry RPC after error@1903513957) -> (continuation of Deferred@609553238 after retry RPC after error@1150480619) -> (continuation of Deferred@1005887413 after retry RPC after error@1615925207) -> (continuation of Deferred@1817356080 after retry RPC after error@1058277369) -> (continuation of Deferred@537013913 after retry RPC after error@1212523257), errback=(continuation of Deferred@413673262 after retry RPC after error@1328826370) -> (continuation of Deferred@939282732 after retry RPC after error@1160032383) -> (continuation of Deferred@485260962 after retry RPC after error@1903513957) -> (continuation of Deferred@609553238 after retry RPC after error@1150480619) -> (continuation of Deferred@1005887413 after retry RPC after error@1615925207) -> (continuation of Deferred@1817356080 after retry RPC after error@1058277369) -> (continuation of Deferred@537013913 after retry RPC after error@1212523257)))
21:04:08.160 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2021741937(state=PENDING, result=null, callback=(continuation of Deferred@413673262 after retry RPC after error@1328826370) -> (continuation of Deferred@939282732 after retry RPC after error@1160032383) -> (continuation of Deferred@485260962 after retry RPC after error@1903513957) -> (continuation of Deferred@609553238 after retry RPC after error@1150480619) -> (continuation of Deferred@1005887413 after retry RPC after error@1615925207) -> (continuation of Deferred@1817356080 after retry RPC after error@1058277369) -> (continuation of Deferred@537013913 after retry RPC after error@1212523257), errback=(continuation of Deferred@413673262 after retry RPC after error@1328826370) -> (continuation of Deferred@939282732 after retry RPC after error@1160032383) -> (continuation of Deferred@485260962 after retry RPC after error@1903513957) -> (continuation of Deferred@609553238 after retry RPC after error@1150480619) -> (continuation of Deferred@1005887413 after retry RPC after error@1615925207) -> (continuation of Deferred@1817356080 after retry RPC after error@1058277369) -> (continuation of Deferred@537013913 after retry RPC after error@1212523257)))
21:04:08.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.219 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.257 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.338 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.340 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1415910869(state=PENDING, result=null, callback=(continuation of Deferred@1368741273 after retry RPC after error@1315088794) -> (continuation of Deferred@855043937 after retry RPC after error@1321312087) -> (continuation of Deferred@897786507 after retry RPC after error@1861641176) -> (continuation of Deferred@1811813504 after retry RPC after error@688326031) -> (continuation of Deferred@87883102 after retry RPC after error@1485420162) -> (continuation of Deferred@1128854279 after retry RPC after error@1391639964), errback=(continuation of Deferred@1368741273 after retry RPC after error@1315088794) -> (continuation of Deferred@855043937 after retry RPC after error@1321312087) -> (continuation of Deferred@897786507 after retry RPC after error@1861641176) -> (continuation of Deferred@1811813504 after retry RPC after error@688326031) -> (continuation of Deferred@87883102 after retry RPC after error@1485420162) -> (continuation of Deferred@1128854279 after retry RPC after error@1391639964)))
21:04:08.340 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1415910869(state=PENDING, result=null, callback=(continuation of Deferred@1368741273 after retry RPC after error@1315088794) -> (continuation of Deferred@855043937 after retry RPC after error@1321312087) -> (continuation of Deferred@897786507 after retry RPC after error@1861641176) -> (continuation of Deferred@1811813504 after retry RPC after error@688326031) -> (continuation of Deferred@87883102 after retry RPC after error@1485420162) -> (continuation of Deferred@1128854279 after retry RPC after error@1391639964), errback=(continuation of Deferred@1368741273 after retry RPC after error@1315088794) -> (continuation of Deferred@855043937 after retry RPC after error@1321312087) -> (continuation of Deferred@897786507 after retry RPC after error@1861641176) -> (continuation of Deferred@1811813504 after retry RPC after error@688326031) -> (continuation of Deferred@87883102 after retry RPC after error@1485420162) -> (continuation of Deferred@1128854279 after retry RPC after error@1391639964)))
21:04:08.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.358 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.540 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.561 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1608027777(state=PENDING, result=null, callback=(continuation of Deferred@2120953633 after retry RPC after error@2132460901) -> (continuation of Deferred@734144618 after retry RPC after error@812975893) -> (continuation of Deferred@398472221 after retry RPC after error@1054407277) -> (continuation of Deferred@73602212 after retry RPC after error@481748626) -> (continuation of Deferred@171989445 after retry RPC after error@1932710258) -> (continuation of Deferred@1869102468 after retry RPC after error@1145160845) -> (continuation of Deferred@1279730609 after retry RPC after error@698077706), errback=(continuation of Deferred@2120953633 after retry RPC after error@2132460901) -> (continuation of Deferred@734144618 after retry RPC after error@812975893) -> (continuation of Deferred@398472221 after retry RPC after error@1054407277) -> (continuation of Deferred@73602212 after retry RPC after error@481748626) -> (continuation of Deferred@171989445 after retry RPC after error@1932710258) -> (continuation of Deferred@1869102468 after retry RPC after error@1145160845) -> (continuation of Deferred@1279730609 after retry RPC after error@698077706)))
21:04:08.562 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1608027777(state=PENDING, result=null, callback=(continuation of Deferred@2120953633 after retry RPC after error@2132460901) -> (continuation of Deferred@734144618 after retry RPC after error@812975893) -> (continuation of Deferred@398472221 after retry RPC after error@1054407277) -> (continuation of Deferred@73602212 after retry RPC after error@481748626) -> (continuation of Deferred@171989445 after retry RPC after error@1932710258) -> (continuation of Deferred@1869102468 after retry RPC after error@1145160845) -> (continuation of Deferred@1279730609 after retry RPC after error@698077706), errback=(continuation of Deferred@2120953633 after retry RPC after error@2132460901) -> (continuation of Deferred@734144618 after retry RPC after error@812975893) -> (continuation of Deferred@398472221 after retry RPC after error@1054407277) -> (continuation of Deferred@73602212 after retry RPC after error@481748626) -> (continuation of Deferred@171989445 after retry RPC after error@1932710258) -> (continuation of Deferred@1869102468 after retry RPC after error@1145160845) -> (continuation of Deferred@1279730609 after retry RPC after error@698077706)))
21:04:08.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.598 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.698 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.778 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.779 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.780 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.780 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2096961704(state=PENDING, result=null, callback=(continuation of Deferred@1553942485 after retry RPC after error@2033875068) -> (continuation of Deferred@2138128358 after retry RPC after error@1801273592) -> (continuation of Deferred@1297748905 after retry RPC after error@1737063398) -> (continuation of Deferred@292681470 after retry RPC after error@271199912) -> (continuation of Deferred@225488991 after retry RPC after error@554351708) -> (continuation of Deferred@1062365317 after retry RPC after error@2118934185), errback=(continuation of Deferred@1553942485 after retry RPC after error@2033875068) -> (continuation of Deferred@2138128358 after retry RPC after error@1801273592) -> (continuation of Deferred@1297748905 after retry RPC after error@1737063398) -> (continuation of Deferred@292681470 after retry RPC after error@271199912) -> (continuation of Deferred@225488991 after retry RPC after error@554351708) -> (continuation of Deferred@1062365317 after retry RPC after error@2118934185)))
21:04:08.781 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@2096961704(state=PENDING, result=null, callback=(continuation of Deferred@1553942485 after retry RPC after error@2033875068) -> (continuation of Deferred@2138128358 after retry RPC after error@1801273592) -> (continuation of Deferred@1297748905 after retry RPC after error@1737063398) -> (continuation of Deferred@292681470 after retry RPC after error@271199912) -> (continuation of Deferred@225488991 after retry RPC after error@554351708) -> (continuation of Deferred@1062365317 after retry RPC after error@2118934185), errback=(continuation of Deferred@1553942485 after retry RPC after error@2033875068) -> (continuation of Deferred@2138128358 after retry RPC after error@1801273592) -> (continuation of Deferred@1297748905 after retry RPC after error@1737063398) -> (continuation of Deferred@292681470 after retry RPC after error@271199912) -> (continuation of Deferred@225488991 after retry RPC after error@554351708) -> (continuation of Deferred@1062365317 after retry RPC after error@2118934185)))
21:04:08.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.798 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:08.979 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@74792192(state=PENDING, result=null, callback=(continuation of Deferred@442509747 after retry RPC after error@1212315483) -> (continuation of Deferred@683379757 after retry RPC after error@84621575) -> (continuation of Deferred@507360676 after retry RPC after error@1168044439) -> (continuation of Deferred@825284589 after retry RPC after error@1831797709) -> (continuation of Deferred@516688729 after retry RPC after error@753685342) -> (continuation of Deferred@458516113 after retry RPC after error@242237277) -> (continuation of Deferred@1889037065 after retry RPC after error@1050718439), errback=(continuation of Deferred@442509747 after retry RPC after error@1212315483) -> (continuation of Deferred@683379757 after retry RPC after error@84621575) -> (continuation of Deferred@507360676 after retry RPC after error@1168044439) -> (continuation of Deferred@825284589 after retry RPC after error@1831797709) -> (continuation of Deferred@516688729 after retry RPC after error@753685342) -> (continuation of Deferred@458516113 after retry RPC after error@242237277) -> (continuation of Deferred@1889037065 after retry RPC after error@1050718439)))
21:04:08.980 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@74792192(state=PENDING, result=null, callback=(continuation of Deferred@442509747 after retry RPC after error@1212315483) -> (continuation of Deferred@683379757 after retry RPC after error@84621575) -> (continuation of Deferred@507360676 after retry RPC after error@1168044439) -> (continuation of Deferred@825284589 after retry RPC after error@1831797709) -> (continuation of Deferred@516688729 after retry RPC after error@753685342) -> (continuation of Deferred@458516113 after retry RPC after error@242237277) -> (continuation of Deferred@1889037065 after retry RPC after error@1050718439), errback=(continuation of Deferred@442509747 after retry RPC after error@1212315483) -> (continuation of Deferred@683379757 after retry RPC after error@84621575) -> (continuation of Deferred@507360676 after retry RPC after error@1168044439) -> (continuation of Deferred@825284589 after retry RPC after error@1831797709) -> (continuation of Deferred@516688729 after retry RPC after error@753685342) -> (continuation of Deferred@458516113 after retry RPC after error@242237277) -> (continuation of Deferred@1889037065 after retry RPC after error@1050718439)))
21:04:08.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:08.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:08.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:08.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:08.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.018 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.020 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.038 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.079 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.118 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.138 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.159 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1840140520(state=PENDING, result=null, callback=(continuation of Deferred@280457413 after retry RPC after error@336002918) -> (continuation of Deferred@216407324 after retry RPC after error@332412280) -> (continuation of Deferred@952782495 after retry RPC after error@462711599) -> (continuation of Deferred@656841660 after retry RPC after error@894370261) -> (continuation of Deferred@1595613013 after retry RPC after error@1735314058) -> (continuation of Deferred@1946572448 after retry RPC after error@965390428) -> (continuation of Deferred@1055715727 after retry RPC after error@793204290), errback=(continuation of Deferred@280457413 after retry RPC after error@336002918) -> (continuation of Deferred@216407324 after retry RPC after error@332412280) -> (continuation of Deferred@952782495 after retry RPC after error@462711599) -> (continuation of Deferred@656841660 after retry RPC after error@894370261) -> (continuation of Deferred@1595613013 after retry RPC after error@1735314058) -> (continuation of Deferred@1946572448 after retry RPC after error@965390428) -> (continuation of Deferred@1055715727 after retry RPC after error@793204290)))
21:04:09.160 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1840140520(state=PENDING, result=null, callback=(continuation of Deferred@280457413 after retry RPC after error@336002918) -> (continuation of Deferred@216407324 after retry RPC after error@332412280) -> (continuation of Deferred@952782495 after retry RPC after error@462711599) -> (continuation of Deferred@656841660 after retry RPC after error@894370261) -> (continuation of Deferred@1595613013 after retry RPC after error@1735314058) -> (continuation of Deferred@1946572448 after retry RPC after error@965390428) -> (continuation of Deferred@1055715727 after retry RPC after error@793204290), errback=(continuation of Deferred@280457413 after retry RPC after error@336002918) -> (continuation of Deferred@216407324 after retry RPC after error@332412280) -> (continuation of Deferred@952782495 after retry RPC after error@462711599) -> (continuation of Deferred@656841660 after retry RPC after error@894370261) -> (continuation of Deferred@1595613013 after retry RPC after error@1735314058) -> (continuation of Deferred@1946572448 after retry RPC after error@965390428) -> (continuation of Deferred@1055715727 after retry RPC after error@793204290)))
21:04:09.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.178 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.179 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.180 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.198 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.238 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.239 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.258 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.259 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.259 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.297 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.377 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.379 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1281645012(state=PENDING, result=null, callback=(continuation of Deferred@519482947 after retry RPC after error@510856384) -> (continuation of Deferred@2120608417 after retry RPC after error@279029312) -> (continuation of Deferred@990807713 after retry RPC after error@1025345548) -> (continuation of Deferred@1828474348 after retry RPC after error@1911715221) -> (continuation of Deferred@1040637572 after retry RPC after error@1862954144) -> (continuation of Deferred@1628954496 after retry RPC after error@53166314) -> (continuation of Deferred@198270197 after retry RPC after error@931286595), errback=(continuation of Deferred@519482947 after retry RPC after error@510856384) -> (continuation of Deferred@2120608417 after retry RPC after error@279029312) -> (continuation of Deferred@990807713 after retry RPC after error@1025345548) -> (continuation of Deferred@1828474348 after retry RPC after error@1911715221) -> (continuation of Deferred@1040637572 after retry RPC after error@1862954144) -> (continuation of Deferred@1628954496 after retry RPC after error@53166314) -> (continuation of Deferred@198270197 after retry RPC after error@931286595)))
21:04:09.380 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@1281645012(state=PENDING, result=null, callback=(continuation of Deferred@519482947 after retry RPC after error@510856384) -> (continuation of Deferred@2120608417 after retry RPC after error@279029312) -> (continuation of Deferred@990807713 after retry RPC after error@1025345548) -> (continuation of Deferred@1828474348 after retry RPC after error@1911715221) -> (continuation of Deferred@1040637572 after retry RPC after error@1862954144) -> (continuation of Deferred@1628954496 after retry RPC after error@53166314) -> (continuation of Deferred@198270197 after retry RPC after error@931286595), errback=(continuation of Deferred@519482947 after retry RPC after error@510856384) -> (continuation of Deferred@2120608417 after retry RPC after error@279029312) -> (continuation of Deferred@990807713 after retry RPC after error@1025345548) -> (continuation of Deferred@1828474348 after retry RPC after error@1911715221) -> (continuation of Deferred@1040637572 after retry RPC after error@1862954144) -> (continuation of Deferred@1628954496 after retry RPC after error@53166314) -> (continuation of Deferred@198270197 after retry RPC after error@931286595)))
21:04:09.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.460 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.478 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.518 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.620 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1959684162(state=PENDING, result=null, callback=(continuation of Deferred@819285866 after retry RPC after error@1697086473) -> (continuation of Deferred@324413517 after retry RPC after error@350080541) -> (continuation of Deferred@241790451 after retry RPC after error@1357942531) -> (continuation of Deferred@1925628176 after retry RPC after error@135353459) -> (continuation of Deferred@1305654903 after retry RPC after error@420388953) -> (continuation of Deferred@1560554537 after retry RPC after error@1605650742) -> (continuation of Deferred@118308281 after retry RPC after error@1802328425), errback=(continuation of Deferred@819285866 after retry RPC after error@1697086473) -> (continuation of Deferred@324413517 after retry RPC after error@350080541) -> (continuation of Deferred@241790451 after retry RPC after error@1357942531) -> (continuation of Deferred@1925628176 after retry RPC after error@135353459) -> (continuation of Deferred@1305654903 after retry RPC after error@420388953) -> (continuation of Deferred@1560554537 after retry RPC after error@1605650742) -> (continuation of Deferred@118308281 after retry RPC after error@1802328425)))
21:04:09.620 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@1959684162(state=PENDING, result=null, callback=(continuation of Deferred@819285866 after retry RPC after error@1697086473) -> (continuation of Deferred@324413517 after retry RPC after error@350080541) -> (continuation of Deferred@241790451 after retry RPC after error@1357942531) -> (continuation of Deferred@1925628176 after retry RPC after error@135353459) -> (continuation of Deferred@1305654903 after retry RPC after error@420388953) -> (continuation of Deferred@1560554537 after retry RPC after error@1605650742) -> (continuation of Deferred@118308281 after retry RPC after error@1802328425), errback=(continuation of Deferred@819285866 after retry RPC after error@1697086473) -> (continuation of Deferred@324413517 after retry RPC after error@350080541) -> (continuation of Deferred@241790451 after retry RPC after error@1357942531) -> (continuation of Deferred@1925628176 after retry RPC after error@135353459) -> (continuation of Deferred@1305654903 after retry RPC after error@420388953) -> (continuation of Deferred@1560554537 after retry RPC after error@1605650742) -> (continuation of Deferred@118308281 after retry RPC after error@1802328425)))
21:04:09.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.758 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.777 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.858 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.860 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@676775882(state=PENDING, result=null, callback=(continuation of Deferred@1750389362 after retry RPC after error@1071006985) -> (continuation of Deferred@635342272 after retry RPC after error@992503334) -> (continuation of Deferred@1973544051 after retry RPC after error@2018646467) -> (continuation of Deferred@699607940 after retry RPC after error@1372742082) -> (continuation of Deferred@724211191 after retry RPC after error@1130237264) -> (continuation of Deferred@1929507252 after retry RPC after error@824136295) -> (continuation of Deferred@1323691166 after retry RPC after error@2111982751), errback=(continuation of Deferred@1750389362 after retry RPC after error@1071006985) -> (continuation of Deferred@635342272 after retry RPC after error@992503334) -> (continuation of Deferred@1973544051 after retry RPC after error@2018646467) -> (continuation of Deferred@699607940 after retry RPC after error@1372742082) -> (continuation of Deferred@724211191 after retry RPC after error@1130237264) -> (continuation of Deferred@1929507252 after retry RPC after error@824136295) -> (continuation of Deferred@1323691166 after retry RPC after error@2111982751)))
21:04:09.860 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@676775882(state=PENDING, result=null, callback=(continuation of Deferred@1750389362 after retry RPC after error@1071006985) -> (continuation of Deferred@635342272 after retry RPC after error@992503334) -> (continuation of Deferred@1973544051 after retry RPC after error@2018646467) -> (continuation of Deferred@699607940 after retry RPC after error@1372742082) -> (continuation of Deferred@724211191 after retry RPC after error@1130237264) -> (continuation of Deferred@1929507252 after retry RPC after error@824136295) -> (continuation of Deferred@1323691166 after retry RPC after error@2111982751), errback=(continuation of Deferred@1750389362 after retry RPC after error@1071006985) -> (continuation of Deferred@635342272 after retry RPC after error@992503334) -> (continuation of Deferred@1973544051 after retry RPC after error@2018646467) -> (continuation of Deferred@699607940 after retry RPC after error@1372742082) -> (continuation of Deferred@724211191 after retry RPC after error@1130237264) -> (continuation of Deferred@1929507252 after retry RPC after error@824136295) -> (continuation of Deferred@1323691166 after retry RPC after error@2111982751)))
21:04:09.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.938 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.957 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:09.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:09.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:09.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:09.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:09.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.037 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.039 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.137 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@69551031(state=PENDING, result=null, callback=(continuation of Deferred@1103902800 after retry RPC after error@1920769103) -> (continuation of Deferred@1305543851 after retry RPC after error@1674038877) -> (continuation of Deferred@1130984653 after retry RPC after error@1432425995) -> (continuation of Deferred@273724496 after retry RPC after error@1303917857) -> (continuation of Deferred@1174404935 after retry RPC after error@410216637) -> (continuation of Deferred@227847639 after retry RPC after error@25788537) -> (continuation of Deferred@993411784 after retry RPC after error@948167571), errback=(continuation of Deferred@1103902800 after retry RPC after error@1920769103) -> (continuation of Deferred@1305543851 after retry RPC after error@1674038877) -> (continuation of Deferred@1130984653 after retry RPC after error@1432425995) -> (continuation of Deferred@273724496 after retry RPC after error@1303917857) -> (continuation of Deferred@1174404935 after retry RPC after error@410216637) -> (continuation of Deferred@227847639 after retry RPC after error@25788537) -> (continuation of Deferred@993411784 after retry RPC after error@948167571)))
21:04:10.138 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:04:10.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.158 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.160 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.179 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.197 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.218 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.357 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.359 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2045630644(state=PENDING, result=null, callback=(continuation of Deferred@5264614 after retry RPC after error@671893657) -> (continuation of Deferred@302779073 after retry RPC after error@95635532) -> (continuation of Deferred@122084216 after retry RPC after error@1304695689) -> (continuation of Deferred@1064063730 after retry RPC after error@537037947) -> (continuation of Deferred@296623640 after retry RPC after error@2082148351) -> (continuation of Deferred@566049100 after retry RPC after error@359139412) -> (continuation of Deferred@1960286783 after retry RPC after error@1343686573), errback=(continuation of Deferred@5264614 after retry RPC after error@671893657) -> (continuation of Deferred@302779073 after retry RPC after error@95635532) -> (continuation of Deferred@122084216 after retry RPC after error@1304695689) -> (continuation of Deferred@1064063730 after retry RPC after error@537037947) -> (continuation of Deferred@296623640 after retry RPC after error@2082148351) -> (continuation of Deferred@566049100 after retry RPC after error@359139412) -> (continuation of Deferred@1960286783 after retry RPC after error@1343686573)))
21:04:10.360 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@2045630644(state=PENDING, result=null, callback=(continuation of Deferred@5264614 after retry RPC after error@671893657) -> (continuation of Deferred@302779073 after retry RPC after error@95635532) -> (continuation of Deferred@122084216 after retry RPC after error@1304695689) -> (continuation of Deferred@1064063730 after retry RPC after error@537037947) -> (continuation of Deferred@296623640 after retry RPC after error@2082148351) -> (continuation of Deferred@566049100 after retry RPC after error@359139412) -> (continuation of Deferred@1960286783 after retry RPC after error@1343686573), errback=(continuation of Deferred@5264614 after retry RPC after error@671893657) -> (continuation of Deferred@302779073 after retry RPC after error@95635532) -> (continuation of Deferred@122084216 after retry RPC after error@1304695689) -> (continuation of Deferred@1064063730 after retry RPC after error@537037947) -> (continuation of Deferred@296623640 after retry RPC after error@2082148351) -> (continuation of Deferred@566049100 after retry RPC after error@359139412) -> (continuation of Deferred@1960286783 after retry RPC after error@1343686573)))
21:04:10.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.379 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.380 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.398 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.418 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.458 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.459 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.538 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.539 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.539 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.539 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.540 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.540 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@454721089(state=PENDING, result=null, callback=(continuation of Deferred@1224388659 after retry RPC after error@1454141138) -> (continuation of Deferred@592916380 after retry RPC after error@1811391937) -> (continuation of Deferred@1989274532 after retry RPC after error@1140773210) -> (continuation of Deferred@1506753910 after retry RPC after error@255869475) -> (continuation of Deferred@1995785380 after retry RPC after error@1127497505) -> (continuation of Deferred@1664126652 after retry RPC after error@1852610979) -> (continuation of Deferred@845164718 after retry RPC after error@1355795700), errback=(continuation of Deferred@1224388659 after retry RPC after error@1454141138) -> (continuation of Deferred@592916380 after retry RPC after error@1811391937) -> (continuation of Deferred@1989274532 after retry RPC after error@1140773210) -> (continuation of Deferred@1506753910 after retry RPC after error@255869475) -> (continuation of Deferred@1995785380 after retry RPC after error@1127497505) -> (continuation of Deferred@1664126652 after retry RPC after error@1852610979) -> (continuation of Deferred@845164718 after retry RPC after error@1355795700)))
21:04:10.540 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, deferred=Deferred@454721089(state=PENDING, result=null, callback=(continuation of Deferred@1224388659 after retry RPC after error@1454141138) -> (continuation of Deferred@592916380 after retry RPC after error@1811391937) -> (continuation of Deferred@1989274532 after retry RPC after error@1140773210) -> (continuation of Deferred@1506753910 after retry RPC after error@255869475) -> (continuation of Deferred@1995785380 after retry RPC after error@1127497505) -> (continuation of Deferred@1664126652 after retry RPC after error@1852610979) -> (continuation of Deferred@845164718 after retry RPC after error@1355795700), errback=(continuation of Deferred@1224388659 after retry RPC after error@1454141138) -> (continuation of Deferred@592916380 after retry RPC after error@1811391937) -> (continuation of Deferred@1989274532 after retry RPC after error@1140773210) -> (continuation of Deferred@1506753910 after retry RPC after error@255869475) -> (continuation of Deferred@1995785380 after retry RPC after error@1127497505) -> (continuation of Deferred@1664126652 after retry RPC after error@1852610979) -> (continuation of Deferred@845164718 after retry RPC after error@1355795700)))
21:04:10.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.578 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.599 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.618 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.638 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.658 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.677 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.678 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.678 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.678 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.799 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@3359720(state=PENDING, result=null, callback=(continuation of Deferred@1904238108 after retry RPC after error@1234742969) -> (continuation of Deferred@1218514579 after retry RPC after error@1377249388) -> (continuation of Deferred@555023500 after retry RPC after error@771056497) -> (continuation of Deferred@1835021770 after retry RPC after error@106902594) -> (continuation of Deferred@11094169 after retry RPC after error@104946966) -> (continuation of Deferred@1784931953 after retry RPC after error@367080027) -> (continuation of Deferred@1606835710 after retry RPC after error@1407386365) -> (continuation of Deferred@650524318 after retry RPC after error@984198135), errback=(continuation of Deferred@1904238108 after retry RPC after error@1234742969) -> (continuation of Deferred@1218514579 after retry RPC after error@1377249388) -> (continuation of Deferred@555023500 after retry RPC after error@771056497) -> (continuation of Deferred@1835021770 after retry RPC after error@106902594) -> (continuation of Deferred@11094169 after retry RPC after error@104946966) -> (continuation of Deferred@1784931953 after retry RPC after error@367080027) -> (continuation of Deferred@1606835710 after retry RPC after error@1407386365) -> (continuation of Deferred@650524318 after retry RPC after error@984198135)))
21:04:10.800 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, deferred=Deferred@3359720(state=PENDING, result=null, callback=(continuation of Deferred@1904238108 after retry RPC after error@1234742969) -> (continuation of Deferred@1218514579 after retry RPC after error@1377249388) -> (continuation of Deferred@555023500 after retry RPC after error@771056497) -> (continuation of Deferred@1835021770 after retry RPC after error@106902594) -> (continuation of Deferred@11094169 after retry RPC after error@104946966) -> (continuation of Deferred@1784931953 after retry RPC after error@367080027) -> (continuation of Deferred@1606835710 after retry RPC after error@1407386365) -> (continuation of Deferred@650524318 after retry RPC after error@984198135), errback=(continuation of Deferred@1904238108 after retry RPC after error@1234742969) -> (continuation of Deferred@1218514579 after retry RPC after error@1377249388) -> (continuation of Deferred@555023500 after retry RPC after error@771056497) -> (continuation of Deferred@1835021770 after retry RPC after error@106902594) -> (continuation of Deferred@11094169 after retry RPC after error@104946966) -> (continuation of Deferred@1784931953 after retry RPC after error@367080027) -> (continuation of Deferred@1606835710 after retry RPC after error@1407386365) -> (continuation of Deferred@650524318 after retry RPC after error@984198135)))
21:04:10.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.818 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.878 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.898 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.917 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.124:46649(127.3.35.124:46649)] connecting to peer
21:04:10.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.3.35.126:35753(127.3.35.126:35753)] connecting to peer
21:04:10.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.35.126:35753(127.3.35.126:35753)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.124:46649(127.3.35.124:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.124:46649: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649
21:04:10.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.35.126:35753: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
21:04:10.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753; will retry
21:04:10.939 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=40, TimeoutTracker(timeout=50000, elapsed=49396), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [6ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [14ms] refreshing cache from master, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [15ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [16ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [16ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [16ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [34ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [54ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [74ms] refreshing cache from master, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [76ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [77ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [77ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [78ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [114ms] refreshing cache from master, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [117ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [174ms] refreshing cache from master, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [176ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [176ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [177ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [177ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [194ms] refreshing cache from master, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [195ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [196ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [196ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [197ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [314ms] refreshing cache from master, [314ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [316ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [316ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [317ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [317ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [636ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [637ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [637ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [637ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1534ms] refreshing cache from master, [1534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1536ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1537ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [1537ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1538ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3334ms] refreshing cache from master, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [3335ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3335ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3336ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3336ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [4594ms] refreshing cache from master, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [4595ms] too many traces: truncated at 100 traces, deferred=Deferred@1180618839(state=PENDING, result=null, callback=(continuation of Deferred@952492241 after retry RPC after error@620555658) -> (continuation of Deferred@2080199883 after retry RPC after error@1267855243) -> (continuation of Deferred@1556438336 after retry RPC after error@521567215) -> (continuation of Deferred@249311178 after retry RPC after error@1249207341) -> (continuation of Deferred@1009520510 after retry RPC after error@273273367) -> (continuation of Deferred@2139360683 after retry RPC after error@514939514) -> (continuation of Deferred@1111159085 after retry RPC after error@482171415) -> (continuation of Deferred@1124857214 after retry RPC after error@1476199919) -> (continuation of Deferred@1113269668 after retry RPC after error@876651034) -> (continuation of Deferred@679215593 after retry RPC after error@387038068) -> (continuation of Deferred@217893431 after retry RPC after error@1033763264) -> (continuation of Deferred@1973867731 after retry RPC after error@1878399752) -> (continuation of Deferred@87148979 after retry RPC after error@1462788908) -> (continuation of Deferred@351479125 after retry RPC after error@1258439320) -> (continuation of Deferred@1089824282 after retry RPC after error@336090155) -> (continuation of Deferred@433848946 after retry RPC after error@1083413843) -> (continuation of Deferred@725479149 after retry RPC after error@2109886125) -> (continuation of Deferred@83107061 after retry RPC after error@1477523509) -> (continuation of Deferred@480247644 after retry RPC after error@1934064964) -> (continuation of Deferred@1467008109 after retry RPC after error@108109350) -> (continuation of Deferred@1702356139 after retry RPC after error@756160540) -> (continuation of Deferred@2081485439 after retry RPC after error@1981465461) -> (continuation of Deferred@747774881 after retry RPC after error@293412055) -> (continuation of Deferred@1787135885 after retry RPC after error@1243934915) -> (continuation of Deferred@2006342925 after retry RPC after error@242510988) -> (continuation of Deferred@1208695584 after retry RPC after error@633628682) -> (continuation of Deferred@1618950026 after retry RPC after error@1304569605) -> (continuation of Deferred@1552013222 after retry RPC after error@1518559236) -> (continuation of Deferred@1138352057 after retry RPC after error@536814280) -> (continuation of Deferred@1301408374 after retry RPC after error@215167104) -> (continuation of Deferred@24773300 after retry RPC after error@1744148785) -> (continuation of Deferred@628432355 after retry RPC after error@817327869) -> (continuation of Deferred@1636410877 after retry RPC after error@721323244) -> (continuation of Deferred@661200469 after retry RPC after error@740449475) -> (continuation of Deferred@11051767 after retry RPC after error@1588563946) -> (continuation of Deferred@828090844 after retry RPC after error@253833072) -> (continuation of Deferred@1247517561 after retry RPC after error@1755946247) -> (continuation of Deferred@72404103 after retry RPC after error@42740178) -> (continuation of Deferred@345701033 after retry RPC after error@822021272), errback=(continuation of Deferred@952492241 after retry RPC after error@620555658) -> (continuation of Deferred@2080199883 after retry RPC after error@1267855243) -> (continuation of Deferred@1556438336 after retry RPC after error@521567215) -> (continuation of Deferred@249311178 after retry RPC after error@1249207341) -> (continuation of Deferred@1009520510 after retry RPC after error@273273367) -> (continuation of Deferred@2139360683 after retry RPC after error@514939514) -> (continuation of Deferred@1111159085 after retry RPC after error@482171415) -> (continuation of Deferred@1124857214 after retry RPC after error@1476199919) -> (continuation of Deferred@1113269668 after retry RPC after error@876651034) -> (continuation of Deferred@679215593 after retry RPC after error@387038068) -> (continuation of Deferred@217893431 after retry RPC after error@1033763264) -> (continuation of Deferred@1973867731 after retry RPC after error@1878399752) -> (continuation of Deferred@87148979 after retry RPC after error@1462788908) -> (continuation of Deferred@351479125 after retry RPC after error@1258439320) -> (continuation of Deferred@1089824282 after retry RPC after error@336090155) -> (continuation of Deferred@433848946 after retry RPC after error@1083413843) -> (continuation of Deferred@725479149 after retry RPC after error@2109886125) -> (continuation of Deferred@83107061 after retry RPC after error@1477523509) -> (continuation of Deferred@480247644 after retry RPC after error@1934064964) -> (continuation of Deferred@1467008109 after retry RPC after error@108109350) -> (continuation of Deferred@1702356139 after retry RPC after error@756160540) -> (continuation of Deferred@2081485439 after retry RPC after error@1981465461) -> (continuation of Deferred@747774881 after retry RPC after error@293412055) -> (continuation of Deferred@1787135885 after retry RPC after error@1243934915) -> (continuation of Deferred@2006342925 after retry RPC after error@242510988) -> (continuation of Deferred@1208695584 after retry RPC after error@633628682) -> (continuation of Deferred@1618950026 after retry RPC after error@1304569605) -> (continuation of Deferred@1552013222 after retry RPC after error@1518559236) -> (continuation of Deferred@1138352057 after retry RPC after error@536814280) -> (continuation of Deferred@1301408374 after retry RPC after error@215167104) -> (continuation of Deferred@24773300 after retry RPC after error@1744148785) -> (continuation of Deferred@628432355 after retry RPC after error@817327869) -> (continuation of Deferred@1636410877 after retry RPC after error@721323244) -> (continuation of Deferred@661200469 after retry RPC after error@740449475) -> (continuation of Deferred@11051767 after retry RPC after error@1588563946) -> (continuation of Deferred@828090844 after retry RPC after error@253833072) -> (continuation of Deferred@1247517561 after retry RPC after error@1755946247) -> (continuation of Deferred@72404103 after retry RPC after error@42740178) -> (continuation of Deferred@345701033 after retry RPC after error@822021272)))
21:04:10.941 [DEBUG - main] (Connection.java:575) [peer 1f5934ecef1445d8840f9064ebcb78c8(127.3.35.65:42735)] disconnecting while in state READY
21:04:10.941 [DEBUG - main] (Connection.java:575) [peer master-127.3.35.126:35753(127.3.35.126:35753)] disconnecting while in state TERMINATED
21:04:10.941 [DEBUG - main] (Connection.java:575) [peer master-127.3.35.124:46649(127.3.35.124:46649)] disconnecting while in state TERMINATED
21:04:10.941 [DEBUG - main] (Connection.java:575) [peer master-127.3.35.125:42683(127.3.35.125:42683)] disconnecting while in state READY
21:04:10.942 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:04:10.943 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 1f5934ecef1445d8840f9064ebcb78c8(127.3.35.65:42735)] handling channelInactive
21:04:10.943 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 1f5934ecef1445d8840f9064ebcb78c8(127.3.35.65:42735)] cleaning up while in state READY due to: connection closed
21:04:10.944 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.3.35.125:42683(127.3.35.125:42683)] handling channelInactive
21:04:10.944 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.35.125:42683(127.3.35.125:42683)] cleaning up while in state READY due to: connection closed
21:04:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:10.943764 3213 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu pid 3508
21:04:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:10.958618 3661 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:11.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.113341 3213 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu with pid 3508
21:04:11.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:11.138537 3828 connection.cc:537] server connection from 127.3.35.65:37367 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:11.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:11.138693 3323 connection.cc:537] server connection from 127.3.35.65:50833 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 21:04:11.138909 3688 connection.cc:537] server connection from 127.3.35.65:47899 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.139103 3213 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu pid 3671
21:04:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.145937 3802 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:11.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.250861 3213 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu with pid 3671
21:04:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.270498 3213 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu pid 3812
21:04:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.277351 3944 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.380812 3213 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu with pid 3812
21:04:11.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.401067 3213 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu pid 3304
21:04:11.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.402293 3376 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 21:04:11.557674 3213 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskrEFwPR/build/asan/bin/kudu with pid 3304
21:04:11.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T21:04:11Z chronyd exiting
21:04:11.719 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=40, TimeoutTracker(timeout=50000, elapsed=49396), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [6ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [14ms] refreshing cache from master, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [15ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [16ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [16ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [16ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [34ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [54ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [74ms] refreshing cache from master, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [76ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [77ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [77ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [78ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [114ms] refreshing cache from master, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [117ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [174ms] refreshing cache from master, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [176ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [176ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [177ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [177ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [194ms] refreshing cache from master, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [195ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [196ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [196ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [197ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [314ms] refreshing cache from master, [314ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [316ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [316ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [317ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [317ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [636ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [637ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [637ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [637ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1534ms] refreshing cache from master, [1534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1536ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1537ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [1537ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1538ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3334ms] refreshing cache from master, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [3335ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3335ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3336ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3336ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [4594ms] refreshing cache from master, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [4595ms] too many traces: truncated at 100 traces, deferred=Deferred@1180618839(state=PENDING, result=null, callback=(continuation of Deferred@952492241 after retry RPC after error@620555658) -> (continuation of Deferred@2080199883 after retry RPC after error@1267855243) -> (continuation of Deferred@1556438336 after retry RPC after error@521567215) -> (continuation of Deferred@249311178 after retry RPC after error@1249207341) -> (continuation of Deferred@1009520510 after retry RPC after error@273273367) -> (continuation of Deferred@2139360683 after retry RPC after error@514939514) -> (continuation of Deferred@1111159085 after retry RPC after error@482171415) -> (continuation of Deferred@1124857214 after retry RPC after error@1476199919) -> (continuation of Deferred@1113269668 after retry RPC after error@876651034) -> (continuation of Deferred@679215593 after retry RPC after error@387038068) -> (continuation of Deferred@217893431 after retry RPC after error@1033763264) -> (continuation of Deferred@1973867731 after retry RPC after error@1878399752) -> (continuation of Deferred@87148979 after retry RPC after error@1462788908) -> (continuation of Deferred@351479125 after retry RPC after error@1258439320) -> (continuation of Deferred@1089824282 after retry RPC after error@336090155) -> (continuation of Deferred@433848946 after retry RPC after error@1083413843) -> (continuation of Deferred@725479149 after retry RPC after error@2109886125) -> (continuation of Deferred@83107061 after retry RPC after error@1477523509) -> (continuation of Deferred@480247644 after retry RPC after error@1934064964) -> (continuation of Deferred@1467008109 after retry RPC after error@108109350) -> (continuation of Deferred@1702356139 after retry RPC after error@756160540) -> (continuation of Deferred@2081485439 after retry RPC after error@1981465461) -> (continuation of Deferred@747774881 after retry RPC after error@293412055) -> (continuation of Deferred@1787135885 after retry RPC after error@1243934915) -> (continuation of Deferred@2006342925 after retry RPC after error@242510988) -> (continuation of Deferred@1208695584 after retry RPC after error@633628682) -> (continuation of Deferred@1618950026 after retry RPC after error@1304569605) -> (continuation of Deferred@1552013222 after retry RPC after error@1518559236) -> (continuation of Deferred@1138352057 after retry RPC after error@536814280) -> (continuation of Deferred@1301408374 after retry RPC after error@215167104) -> (continuation of Deferred@24773300 after retry RPC after error@1744148785) -> (continuation of Deferred@628432355 after retry RPC after error@817327869) -> (continuation of Deferred@1636410877 after retry RPC after error@721323244) -> (continuation of Deferred@661200469 after retry RPC after error@740449475) -> (continuation of Deferred@11051767 after retry RPC after error@1588563946) -> (continuation of Deferred@828090844 after retry RPC after error@253833072) -> (continuation of Deferred@1247517561 after retry RPC after error@1755946247) -> (continuation of Deferred@72404103 after retry RPC after error@42740178) -> (continuation of Deferred@345701033 after retry RPC after error@822021272), errback=(continuation of Deferred@952492241 after retry RPC after error@620555658) -> (continuation of Deferred@2080199883 after retry RPC after error@1267855243) -> (continuation of Deferred@1556438336 after retry RPC after error@521567215) -> (continuation of Deferred@249311178 after retry RPC after error@1249207341) -> (continuation of Deferred@1009520510 after retry RPC after error@273273367) -> (continuation of Deferred@2139360683 after retry RPC after error@514939514) -> (continuation of Deferred@1111159085 after retry RPC after error@482171415) -> (continuation of Deferred@1124857214 after retry RPC after error@1476199919) -> (continuation of Deferred@1113269668 after retry RPC after error@876651034) -> (continuation of Deferred@679215593 after retry RPC after error@387038068) -> (continuation of Deferred@217893431 after retry RPC after error@1033763264) -> (continuation of Deferred@1973867731 after retry RPC after error@1878399752) -> (continuation of Deferred@87148979 after retry RPC after error@1462788908) -> (continuation of Deferred@351479125 after retry RPC after error@1258439320) -> (continuation of Deferred@1089824282 after retry RPC after error@336090155) -> (continuation of Deferred@433848946 after retry RPC after error@1083413843) -> (continuation of Deferred@725479149 after retry RPC after error@2109886125) -> (continuation of Deferred@83107061 after retry RPC after error@1477523509) -> (continuation of Deferred@480247644 after retry RPC after error@1934064964) -> (continuation of Deferred@1467008109 after retry RPC after error@108109350) -> (continuation of Deferred@1702356139 after retry RPC after error@756160540) -> (continuation of Deferred@2081485439 after retry RPC after error@1981465461) -> (continuation of Deferred@747774881 after retry RPC after error@293412055) -> (continuation of Deferred@1787135885 after retry RPC after error@1243934915) -> (continuation of Deferred@2006342925 after retry RPC after error@242510988) -> (continuation of Deferred@1208695584 after retry RPC after error@633628682) -> (continuation of Deferred@1618950026 after retry RPC after error@1304569605) -> (continuation of Deferred@1552013222 after retry RPC after error@1518559236) -> (continuation of Deferred@1138352057 after retry RPC after error@536814280) -> (continuation of Deferred@1301408374 after retry RPC after error@215167104) -> (continuation of Deferred@24773300 after retry RPC after error@1744148785) -> (continuation of Deferred@628432355 after retry RPC after error@817327869) -> (continuation of Deferred@1636410877 after retry RPC after error@721323244) -> (continuation of Deferred@661200469 after retry RPC after error@740449475) -> (continuation of Deferred@11051767 after retry RPC after error@1588563946) -> (continuation of Deferred@828090844 after retry RPC after error@253833072) -> (continuation of Deferred@1247517561 after retry RPC after error@1755946247) -> (continuation of Deferred@72404103 after retry RPC after error@42740178) -> (continuation of Deferred@345701033 after retry RPC after error@822021272)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.35.126:35753
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=40, TimeoutTracker(timeout=50000, elapsed=49396), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [6ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [14ms] refreshing cache from master, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [15ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [16ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [16ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [16ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [34ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [36ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [54ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [56ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [74ms] refreshing cache from master, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [76ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [77ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [77ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [78ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [114ms] refreshing cache from master, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [116ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [117ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [174ms] refreshing cache from master, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [176ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [176ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [177ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [177ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [194ms] refreshing cache from master, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [195ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [196ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [196ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [197ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [314ms] refreshing cache from master, [314ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [316ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [316ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [317ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [317ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [636ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [637ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [637ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [637ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1534ms] refreshing cache from master, [1534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [1536ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [1537ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [1537ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [1538ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3334ms] refreshing cache from master, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [3334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [3335ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.124:46649: Network error: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649, [3335ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.126:35753: Network error: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [3336ms] Sub RPC ConnectToMaster: received response from server master-127.3.35.125:42683: OK, [3336ms] delaying RPC due to: Service unavailable: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753, [4594ms] refreshing cache from master, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.124:46649, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.125:42683, [4594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.35.126:35753, [4595ms] too many traces: truncated at 100 traces, deferred=Deferred@1180618839(state=PENDING, result=null, callback=(continuation of Deferred@952492241 after retry RPC after error@620555658) -> (continuation of Deferred@2080199883 after retry RPC after error@1267855243) -> (continuation of Deferred@1556438336 after retry RPC after error@521567215) -> (continuation of Deferred@249311178 after retry RPC after error@1249207341) -> (continuation of Deferred@1009520510 after retry RPC after error@273273367) -> (continuation of Deferred@2139360683 after retry RPC after error@514939514) -> (continuation of Deferred@1111159085 after retry RPC after error@482171415) -> (continuation of Deferred@1124857214 after retry RPC after error@1476199919) -> (continuation of Deferred@1113269668 after retry RPC after error@876651034) -> (continuation of Deferred@679215593 after retry RPC after error@387038068) -> (continuation of Deferred@217893431 after retry RPC after error@1033763264) -> (continuation of Deferred@1973867731 after retry RPC after error@1878399752) -> (continuation of Deferred@87148979 after retry RPC after error@1462788908) -> (continuation of Deferred@351479125 after retry RPC after error@1258439320) -> (continuation of Deferred@1089824282 after retry RPC after error@336090155) -> (continuation of Deferred@433848946 after retry RPC after error@1083413843) -> (continuation of Deferred@725479149 after retry RPC after error@2109886125) -> (continuation of Deferred@83107061 after retry RPC after error@1477523509) -> (continuation of Deferred@480247644 after retry RPC after error@1934064964) -> (continuation of Deferred@1467008109 after retry RPC after error@108109350) -> (continuation of Deferred@1702356139 after retry RPC after error@756160540) -> (continuation of Deferred@2081485439 after retry RPC after error@1981465461) -> (continuation of Deferred@747774881 after retry RPC after error@293412055) -> (continuation of Deferred@1787135885 after retry RPC after error@1243934915) -> (continuation of Deferred@2006342925 after retry RPC after error@242510988) -> (continuation of Deferred@1208695584 after retry RPC after error@633628682) -> (continuation of Deferred@1618950026 after retry RPC after error@1304569605) -> (continuation of Deferred@1552013222 after retry RPC after error@1518559236) -> (continuation of Deferred@1138352057 after retry RPC after error@536814280) -> (continuation of Deferred@1301408374 after retry RPC after error@215167104) -> (continuation of Deferred@24773300 after retry RPC after error@1744148785) -> (continuation of Deferred@628432355 after retry RPC after error@817327869) -> (continuation of Deferred@1636410877 after retry RPC after error@721323244) -> (continuation of Deferred@661200469 after retry RPC after error@740449475) -> (continuation of Deferred@11051767 after retry RPC after error@1588563946) -> (continuation of Deferred@828090844 after retry RPC after error@253833072) -> (continuation of Deferred@1247517561 after retry RPC after error@1755946247) -> (continuation of Deferred@72404103 after retry RPC after error@42740178) -> (continuation of Deferred@345701033 after retry RPC after error@822021272), errback=(continuation of Deferred@952492241 after retry RPC after error@620555658) -> (continuation of Deferred@2080199883 after retry RPC after error@1267855243) -> (continuation of Deferred@1556438336 after retry RPC after error@521567215) -> (continuation of Deferred@249311178 after retry RPC after error@1249207341) -> (continuation of Deferred@1009520510 after retry RPC after error@273273367) -> (continuation of Deferred@2139360683 after retry RPC after error@514939514) -> (continuation of Deferred@1111159085 after retry RPC after error@482171415) -> (continuation of Deferred@1124857214 after retry RPC after error@1476199919) -> (continuation of Deferred@1113269668 after retry RPC after error@876651034) -> (continuation of Deferred@679215593 after retry RPC after error@387038068) -> (continuation of Deferred@217893431 after retry RPC after error@1033763264) -> (continuation of Deferred@1973867731 after retry RPC after error@1878399752) -> (continuation of Deferred@87148979 after retry RPC after error@1462788908) -> (continuation of Deferred@351479125 after retry RPC after error@1258439320) -> (continuation of Deferred@1089824282 after retry RPC after error@336090155) -> (continuation of Deferred@433848946 after retry RPC after error@1083413843) -> (continuation of Deferred@725479149 after retry RPC after error@2109886125) -> (continuation of Deferred@83107061 after retry RPC after error@1477523509) -> (continuation of Deferred@480247644 after retry RPC after error@1934064964) -> (continuation of Deferred@1467008109 after retry RPC after error@108109350) -> (continuation of Deferred@1702356139 after retry RPC after error@756160540) -> (continuation of Deferred@2081485439 after retry RPC after error@1981465461) -> (continuation of Deferred@747774881 after retry RPC after error@293412055) -> (continuation of Deferred@1787135885 after retry RPC after error@1243934915) -> (continuation of Deferred@2006342925 after retry RPC after error@242510988) -> (continuation of Deferred@1208695584 after retry RPC after error@633628682) -> (continuation of Deferred@1618950026 after retry RPC after error@1304569605) -> (continuation of Deferred@1552013222 after retry RPC after error@1518559236) -> (continuation of Deferred@1138352057 after retry RPC after error@536814280) -> (continuation of Deferred@1301408374 after retry RPC after error@215167104) -> (continuation of Deferred@24773300 after retry RPC after error@1744148785) -> (continuation of Deferred@628432355 after retry RPC after error@817327869) -> (continuation of Deferred@1636410877 after retry RPC after error@721323244) -> (continuation of Deferred@661200469 after retry RPC after error@740449475) -> (continuation of Deferred@11051767 after retry RPC after error@1588563946) -> (continuation of Deferred@828090844 after retry RPC after error@253833072) -> (continuation of Deferred@1247517561 after retry RPC after error@1755946247) -> (continuation of Deferred@72404103 after retry RPC after error@42740178) -> (continuation of Deferred@345701033 after retry RPC after error@822021272)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.35.124:46649,127.3.35.125:42683,127.3.35.126:35753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.124:46649(127.3.35.124:46649): Connection refused: /127.3.35.124:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.35.126:35753(127.3.35.126:35753): Connection refused: /127.3.35.126:35753
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.35.126:35753
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)